CatalogManagerConfigurations/MasterStressTest.Test/3: WARNING: ThreadSanitizer: data race (pid=430)  Atomic write of size 8 at 0x7f7b2a3758b8 by thread T32:
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (master-stress-test+0x334de8)
    #1 unsigned long* google::glog_internal_namespace_::sync_val_compare_and_swap<unsigned long*>(unsigned long**, unsigned long*, unsigned long*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/utilities.h:172:10 (libglog.so.1+0x2b285)
    #2 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/signalhandler.cc:296:7 (libglog.so.1+0x2b285)
    #3 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (master-stress-test+0x2dadff)
    #4 google::LogMessage::Fail() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1946:3 (libglog.so.1+0x18da3)
    #5 google::LogMessage::SendToLog() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1920:5 (libglog.so.1+0x18da3)
    #6 google::LogMessage::Flush() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1777:5 (libglog.so.1+0x1994f)
    #7 google::LogMessageFatal::~LogMessageFatal() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:2557:5 (libglog.so.1+0x1ea8a)
    #8 kudu::MasterStressTest::AlterTableThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:328:7 (master-stress-test+0x35d525)
    #9 decltype(*(std::__1::forward<kudu::MasterStressTest_Test_Test*>(fp0)).*fp()) std::__1::__invoke<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846:1 (master-stress-test+0x38ae0d)
    #10 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (master-stress-test+0x38acc0)
    #11 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*> >(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (master-stress-test+0x38a40a)

  Previous read of size 8 at 0x7f7b2a3758b8 by thread T23:
    #0 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/signalhandler.cc:302:38 (libglog.so.1+0x2ca90)
    #1 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (master-stress-test+0x2dadff)
    #2 google::LogMessage::Fail() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1946:3 (libglog.so.1+0x18da3)
    #3 google::LogMessage::SendToLog() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1920:5 (libglog.so.1+0x18da3)
    #4 google::LogMessage::Flush() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1777:5 (libglog.so.1+0x1994f)
    #5 google::LogMessageFatal::~LogMessageFatal() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:2557:5 (libglog.so.1+0x1ea8a)
    #6 kudu::MasterStressTest::CreateTableThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc (master-stress-test+0x35cf33)
    #7 decltype(*(std::__1::forward<kudu::MasterStressTest_Test_Test*>(fp0)).*fp()) std::__1::__invoke<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846:1 (master-stress-test+0x38ae0d)
    #8 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (master-stress-test+0x38acc0)
    #9 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*> >(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (master-stress-test+0x38a40a)
    @     0x7f7b2a14a950 google::LogMessage::Flush() at ??:0

  Location is global 'google::(anonymous namespace)::g_entered_thread_id_pointer' of size 8 at 0x7f7b2a3758b8 (libglog.so.1+0x0000002448b8)

  Thread T32 (tid=3830, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (master-stress-test+0x2de775)
    #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (master-stress-test+0x38a39b)
    #2 std::__1::thread::thread<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (master-stress-test+0x38a0f7)
    #3 void std::__1::allocator<std::__1::thread>::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1809:31 (master-stress-test+0x389ffc)
    #4 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::__construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1687:21 (master-stress-test+0x389f93)
    #5 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1538:14 (master-stress-test+0x389ed3)
    #6 void std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::__construct_one_at_end<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:926:5 (master-stress-test+0x389bde)
    #7 std::__1::thread& std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::emplace_back<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1687:9 (master-stress-test+0x35c98a)
    #8 kudu::MasterStressTest_Test_Test::TestBody() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:560:13 (master-stress-test+0x35b876)
    #9 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
    #10 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
    #11 testing::Test::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674:5 (libgtest.so.1.12.1+0x42a31)
    #12 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43d48)
    #13 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
    #14 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
    #15 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
    #16 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
    #17 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
    #18 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (master-stress-test+0x38d1db)
    #19 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (master-stress-test+0x38c0dc)

  Thread T23 (tid=3825, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (master-stress-test+0x2de775)
    #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (master-stress-test+0x38a39b)
    #2 std::__1::thread::thread<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (master-stress-test+0x38a0f7)
    #3 void std::__1::allocator<std::__1::thread>::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1809:31 (master-stress-test+0x389ffc)
    #4 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::__construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1687:21 (master-stress-test+0x389f93)
    #5 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1538:14 (master-stress-test+0x389ed3)
    #6 void std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::__emplace_back_slow_path<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1670:5 (master-stress-test+0x389d0e)
    #7 std::__1::thread& std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::emplace_back<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1690:9 (master-stress-test+0x35c99a)
    #8 kudu::MasterStressTest_Test_Test::TestBody() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:557:13 (master-stress-test+0x35b806)
    #9 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
    #10 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
    #11 testing::Test::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674:5 (libgtest.so.1.12.1+0x42a31)
    #12 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43d48)
    #13 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
    #14 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
    #15 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
    #16 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
    #17 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
    #18 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (master-stress-test+0x38d1db)
    #19 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (master-stress-test+0x38c0dc)

Note: This is test shard 1 of 3.
[==========] Running 2 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 2 tests from CatalogManagerConfigurations/MasterStressTest
[ RUN      ] CatalogManagerConfigurations/MasterStressTest.Test/0
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20250628 07:51:56.502781   430 test_util.cc:276] Using random seed: 295217784
2025-06-28T07:51:56Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
2025-06-28T07:51:56Z Disabled control of system clock
I20250628 07:51:56.569245   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:37209
--webserver_interface=127.0.107.190
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20250628 07:51:56.873639   439 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:51:56.874207   439 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:51:56.874619   439 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:51:56.911288   439 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:51:56.911583   439 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:51:56.911823   439 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:51:56.912026   439 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:51:56.947903   439 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:37209
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.107.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:51:56.949184   439 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:51:56.950744   439 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:51:56.965708   446 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:51:56.966311   445 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:51:56.968271   448 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:51:56.968854   439 server_base.cc:1048] running on GCE node
I20250628 07:51:58.124361   439 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:51:58.126914   439 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:51:58.128245   439 hybrid_clock.cc:648] HybridClock initialized: now 1751097118128196 us; error 68 us; skew 500 ppm
I20250628 07:51:58.129004   439 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:51:58.139839   439 webserver.cc:469] Webserver started at http://127.0.107.190:42501/ using document root <none> and password file <none>
I20250628 07:51:58.140731   439 fs_manager.cc:362] Metadata directory not provided
I20250628 07:51:58.140937   439 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:51:58.141389   439 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:51:58.145720   439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/instance:
uuid: "5fd67221fa5f424ea0f83e582629d276"
format_stamp: "Formatted at 2025-06-28 07:51:58 on dist-test-slave-kfnw"
I20250628 07:51:58.146759   439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal/instance:
uuid: "5fd67221fa5f424ea0f83e582629d276"
format_stamp: "Formatted at 2025-06-28 07:51:58 on dist-test-slave-kfnw"
I20250628 07:51:58.154613   439 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
I20250628 07:51:58.159914   455 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:51:58.160820   439 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
I20250628 07:51:58.161161   439 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
uuid: "5fd67221fa5f424ea0f83e582629d276"
format_stamp: "Formatted at 2025-06-28 07:51:58 on dist-test-slave-kfnw"
I20250628 07:51:58.161474   439 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:51:58.215571   439 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:51:58.216995   439 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:51:58.217453   439 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:51:58.295197   439 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.190:37209
I20250628 07:51:58.295274   506 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.190:37209 every 8 connection(s)
I20250628 07:51:58.297869   439 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
I20250628 07:51:58.303135   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 439
I20250628 07:51:58.303659   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal/instance
I20250628 07:51:58.304694   507 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:51:58.309175   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.189:33023
--webserver_interface=127.0.107.189
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:51:58.318863   507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:51:58.343647   507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:51:58.345827   459 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
W20250628 07:51:58.349886   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
I20250628 07:51:58.399456   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 1
W20250628 07:51:58.403851   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
I20250628 07:51:58.472366   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 2
W20250628 07:51:58.475976   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
I20250628 07:51:58.567464   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 3
W20250628 07:51:58.571130   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
W20250628 07:51:58.609540   509 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:51:58.610071   509 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:51:58.610453   509 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:51:58.640728   509 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:51:58.641021   509 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:51:58.641240   509 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:51:58.641433   509 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:51:58.677039   509 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.189:33023
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
--webserver_interface=127.0.107.189
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:51:58.678710   509 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:51:58.680629   509 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:51:58.696748   518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:51:58.714565   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 4
W20250628 07:51:58.718111   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
I20250628 07:51:59.017498   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 5
W20250628 07:51:59.024499   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
I20250628 07:51:59.572196   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 6
W20250628 07:51:59.584530   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:33023: Network error: Client connection negotiation failed: client connection to 127.0.107.189:33023: connect: Connection refused (error 111)
W20250628 07:51:58.697019   519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:51:58.697883   521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:51:59.923017   520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1224 milliseconds
I20250628 07:51:59.923120   509 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:51:59.924273   509 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:51:59.927211   509 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:51:59.928545   509 hybrid_clock.cc:648] HybridClock initialized: now 1751097119928487 us; error 68 us; skew 500 ppm
I20250628 07:51:59.929365   509 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:51:59.935539   509 webserver.cc:469] Webserver started at http://127.0.107.189:37733/ using document root <none> and password file <none>
I20250628 07:51:59.936436   509 fs_manager.cc:362] Metadata directory not provided
I20250628 07:51:59.936645   509 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:51:59.937116   509 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:51:59.941040   509 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data/instance:
uuid: "023ed4cb582a44509bccf42b52c494d7"
format_stamp: "Formatted at 2025-06-28 07:51:59 on dist-test-slave-kfnw"
I20250628 07:51:59.942155   509 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal/instance:
uuid: "023ed4cb582a44509bccf42b52c494d7"
format_stamp: "Formatted at 2025-06-28 07:51:59 on dist-test-slave-kfnw"
I20250628 07:51:59.949021   509 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.001s
I20250628 07:51:59.954833   530 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:51:59.955821   509 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
I20250628 07:51:59.956123   509 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal
uuid: "023ed4cb582a44509bccf42b52c494d7"
format_stamp: "Formatted at 2025-06-28 07:51:59 on dist-test-slave-kfnw"
I20250628 07:51:59.956441   509 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:00.005708   509 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:00.007112   509 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:00.007534   509 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:00.083506   509 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.189:33023
I20250628 07:52:00.083575   581 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.189:33023 every 8 connection(s)
I20250628 07:52:00.086249   509 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
I20250628 07:52:00.092370   582 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:00.096756   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 509
I20250628 07:52:00.097291   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-1/wal/instance
I20250628 07:52:00.100589   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:42677
--webserver_interface=127.0.107.188
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:52:00.107400   582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:52:00.132722   582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:52:00.146034   582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:52:00.148001   534 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
W20250628 07:52:00.152626   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.202169   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 1
W20250628 07:52:00.206351   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.274832   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 2
W20250628 07:52:00.279369   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.370785   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 3
W20250628 07:52:00.375065   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
W20250628 07:52:00.410601   584 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:52:00.411213   584 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:52:00.411666   584 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:52:00.442308   584 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:52:00.442631   584 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:52:00.442878   584 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:52:00.443121   584 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:52:00.477643   584 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:42677
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.107.188
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:52:00.479039   584 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:52:00.480571   584 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:52:00.497413   594 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:00.518534   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 4
W20250628 07:52:00.522833   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.645185   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } attempt: 7
I20250628 07:52:00.661130   507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:52:00.668970   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.727514   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 1
W20250628 07:52:00.731675   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.813045   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 2
W20250628 07:52:00.817329   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.822194   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 5
W20250628 07:52:00.826810   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:00.902675   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 3
W20250628 07:52:00.908385   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:01.048791   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 4
W20250628 07:52:01.053870   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:01.337222   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 5
W20250628 07:52:01.341650   507 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
I20250628 07:52:01.374177   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 6
W20250628 07:52:01.381074   582 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:42677: Network error: Client connection negotiation failed: client connection to 127.0.107.188:42677: connect: Connection refused (error 111)
W20250628 07:52:00.497794   597 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:00.498179   584 server_base.cc:1048] running on GCE node
W20250628 07:52:00.497413   595 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:01.685649   584 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:52:01.688158   584 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:52:01.689491   584 hybrid_clock.cc:648] HybridClock initialized: now 1751097121689447 us; error 58 us; skew 500 ppm
I20250628 07:52:01.690263   584 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:52:01.696615   584 webserver.cc:469] Webserver started at http://127.0.107.188:40861/ using document root <none> and password file <none>
I20250628 07:52:01.697582   584 fs_manager.cc:362] Metadata directory not provided
I20250628 07:52:01.697794   584 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:52:01.698235   584 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:52:01.702580   584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data/instance:
uuid: "992ac4c27fb744deb1edf7765553ef08"
format_stamp: "Formatted at 2025-06-28 07:52:01 on dist-test-slave-kfnw"
I20250628 07:52:01.703670   584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal/instance:
uuid: "992ac4c27fb744deb1edf7765553ef08"
format_stamp: "Formatted at 2025-06-28 07:52:01 on dist-test-slave-kfnw"
I20250628 07:52:01.710808   584 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.000s	sys 0.008s
I20250628 07:52:01.716504   609 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:01.717557   584 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
I20250628 07:52:01.717851   584 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal
uuid: "992ac4c27fb744deb1edf7765553ef08"
format_stamp: "Formatted at 2025-06-28 07:52:01 on dist-test-slave-kfnw"
I20250628 07:52:01.718164   584 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:01.778828   584 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:01.780660   584 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:01.781108   584 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:01.857580   584 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.188:42677
I20250628 07:52:01.857648   660 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.188:42677 every 8 connection(s)
I20250628 07:52:01.860002   584 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
I20250628 07:52:01.865295   661 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:01.867215   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 584
I20250628 07:52:01.867569   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-2/wal/instance
I20250628 07:52:01.877553   661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:52:01.893992   507 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 6
I20250628 07:52:01.930575   661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:52:01.941929   507 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Bootstrap starting.
I20250628 07:52:01.943138   661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:52:01.949187   507 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:01.951560   507 log.cc:826] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:01.958139   507 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: No bootstrap required, opened a new log
I20250628 07:52:01.961454   661 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Bootstrap starting.
I20250628 07:52:01.969043   661 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:01.971149   661 log.cc:826] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:01.977403   661 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: No bootstrap required, opened a new log
I20250628 07:52:01.978219   507 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:01.978832   507 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:01.979107   507 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5fd67221fa5f424ea0f83e582629d276, State: Initialized, Role: FOLLOWER
I20250628 07:52:01.979776   507 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:01.983035   673 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:01.983846   673 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:01.985172   507 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:52:01.998826   661 raft_consensus.cc:357] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:01.999817   661 raft_consensus.cc:383] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:02.000140   661 raft_consensus.cc:738] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 992ac4c27fb744deb1edf7765553ef08, State: Initialized, Role: FOLLOWER
I20250628 07:52:02.001405   661 consensus_queue.cc:260] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.006214   682 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.007098   682 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.008368   661 sys_catalog.cc:564] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: configured and running, proceeding with master startup.
W20250628 07:52:02.025717   685 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:52:02.026131   685 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:52:02.036248   615 tablet.cc:2378] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:02.017896   439 master_runner.cc:186] Error getting master registration for 127.0.107.189:33023: OK, instance_id { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" instance_seqno: 1751097120062542 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20250628 07:52:02.052301   439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:52:02.060372   697 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:52:02.060710   697 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:52:02.062065   584 master_runner.cc:186] Error getting master registration for 127.0.107.189:33023: OK, instance_id { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" instance_seqno: 1751097120062542 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20250628 07:52:02.084267   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:02.177170   682 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:02.177769   682 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.180655   682 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:02.181746   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:02.182591   482 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 0.
I20250628 07:52:02.183849   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08; no voters: 
I20250628 07:52:02.184775   682 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:02.185199   682 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:02.185581   682 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:02.193010   682 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.195371   682 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 1 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:02.195647   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:02.196177   482 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:02.205161   482 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 1.
I20250628 07:52:02.214116   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08; no voters: 
I20250628 07:52:02.215009   682 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:02.216799   682 raft_consensus.cc:695] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 LEADER]: Becoming Leader. State: Replica: 992ac4c27fb744deb1edf7765553ef08, State: Running, Role: LEADER
I20250628 07:52:02.217816   682 consensus_queue.cc:237] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
W20250628 07:52:02.227638   461 tablet.cc:2378] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:02.230197   699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 992ac4c27fb744deb1edf7765553ef08. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.230991   699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:02.236930   701 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:52:02.244398   701 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:52:02.272784   482 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Refusing update from remote peer 992ac4c27fb744deb1edf7765553ef08: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:02.274395   699 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:02.319460   673 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 992ac4c27fb744deb1edf7765553ef08. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.320192   673 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.333820   702 mvcc.cc:204] Tried to move back new op lower bound from 7172493812810059776 to 7172493812614660096. Current Snapshot: MvccSnapshot[applied={T|T < 7172493812810059776}]
I20250628 07:52:02.382731   673 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.383400   673 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.392151   701 catalog_manager.cc:1349] Generated new cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:02.392424   701 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:52:02.393360   699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.394419   699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:02.436564   701 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:52:02.438997   701 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:52:02.441679   582 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } attempt: 7
I20250628 07:52:02.467937   701 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Generated new TSK 0
I20250628 07:52:02.472457   701 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:52:02.496594   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.129:0
--local_ip_for_outbound_sockets=127.0.107.129
--webserver_interface=127.0.107.129
--webserver_port=0
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:52:02.504921   582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Bootstrap starting.
I20250628 07:52:02.512298   582 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:02.514909   582 log.cc:826] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:02.547883   582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: No bootstrap required, opened a new log
I20250628 07:52:02.620378   582 raft_consensus.cc:357] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.622689   582 raft_consensus.cc:383] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:02.623551   582 raft_consensus.cc:738] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 023ed4cb582a44509bccf42b52c494d7, State: Initialized, Role: FOLLOWER
I20250628 07:52:02.625995   582 consensus_queue.cc:260] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.648561   557 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:02.650002   556 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7"
I20250628 07:52:02.652271   555 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:02.659731   555 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Refusing update from remote peer 992ac4c27fb744deb1edf7765553ef08: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:02.665215   582 sys_catalog.cc:564] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:52:02.665654   710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.669910   710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.672192   699 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Connected to new peer: Peer: permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:02.766713   710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 992ac4c27fb744deb1edf7765553ef08. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.770308   710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.769544   717 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:02.770836   717 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:02.805137   727 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20250628 07:52:02.987432   732 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:02.988173   732 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:02.992547   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:02.996060   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:02.997738   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:03.000227   732 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:03.001194   732 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20250628 07:52:03.009174   708 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:52:03.009805   708 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:52:03.010753   708 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20250628 07:52:03.034046   685 catalog_manager.cc:1261] Loaded cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:03.034411   685 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: loading cluster ID for follower catalog manager: success
I20250628 07:52:03.042007   685 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: acquiring CA information for follower catalog manager: success
I20250628 07:52:03.047607   685 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
W20250628 07:52:03.068022   708 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:52:03.069590   708 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.129
I20250628 07:52:03.131608   708 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.0.107.129
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.129
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:52:03.133329   708 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:52:03.135474   708 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:52:03.173753   736 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:03.173827   738 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:03.174589   738 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:03.552742   745 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:03.553566   745 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:03.557165   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:03.560945   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:03.564901   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:03.573661   745 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:03.573904   746 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:03.691988   747 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:03.692910   747 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:03.696494   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:03.697783   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:03.703687   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7; no voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:03.712671   747 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:03.714020   747 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:03.829245   727 catalog_manager.cc:1261] Loaded cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:03.829663   727 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: loading cluster ID for follower catalog manager: success
I20250628 07:52:03.852331   727 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: acquiring CA information for follower catalog manager: success
I20250628 07:52:03.894805   727 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:52:05.229897   750 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:05.230743   750 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:05.243786   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:05.244071   750 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:05.245621   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:05.247166   534 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7; no voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:05.247874   750 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20250628 07:52:04.564518   734 debug-util.cc:398] Leaking SignalData structure 0x7b08000186a0 after lost signal to thread 708
W20250628 07:52:05.589582   734 kernel_stack_watchdog.cc:198] Thread 708 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 381ms:
Kernel stack:
(could not read kernel stack)
User stack:
<Timed out: thread did not respond: maybe it is blocking signals>
W20250628 07:52:03.181504   737 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:52:05.590976   741 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2407 milliseconds
W20250628 07:52:05.607081   708 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 2.426s	user 0.010s	sys 0.007s
W20250628 07:52:05.609218   708 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 2.428s	user 0.010s	sys 0.007s
W20250628 07:52:05.609812   743 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:05.609825   708 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:52:05.611620   708 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:52:05.619817   708 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:52:05.621299   708 hybrid_clock.cc:648] HybridClock initialized: now 1751097125621263 us; error 41 us; skew 500 ppm
I20250628 07:52:05.622432   708 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:52:05.646095   708 webserver.cc:469] Webserver started at http://127.0.107.129:32779/ using document root <none> and password file <none>
I20250628 07:52:05.647521   708 fs_manager.cc:362] Metadata directory not provided
I20250628 07:52:05.647863   708 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:52:05.648722   708 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:52:05.655714   708 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data/instance:
uuid: "6005c44dfe4848f18a304621cd49e934"
format_stamp: "Formatted at 2025-06-28 07:52:05 on dist-test-slave-kfnw"
I20250628 07:52:05.657320   708 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/instance:
uuid: "6005c44dfe4848f18a304621cd49e934"
format_stamp: "Formatted at 2025-06-28 07:52:05 on dist-test-slave-kfnw"
I20250628 07:52:05.672647   708 fs_manager.cc:696] Time spent creating directory manager: real 0.014s	user 0.008s	sys 0.004s
I20250628 07:52:05.685560   755 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:05.686962   708 fs_manager.cc:730] Time spent opening block manager: real 0.010s	user 0.007s	sys 0.004s
I20250628 07:52:05.687404   708 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal
uuid: "6005c44dfe4848f18a304621cd49e934"
format_stamp: "Formatted at 2025-06-28 07:52:05 on dist-test-slave-kfnw"
I20250628 07:52:05.687893   708 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:05.795810   760 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:05.797919   760 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:05.804484   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:05.805632   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:05.807600   534 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7; no voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:05.813848   760 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:05.814294   760 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:05.827047   708 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:05.828920   708 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:05.829471   708 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:05.833330   708 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:52:05.840348   708 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:52:05.840626   708 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:05.840919   708 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:52:05.841172   708 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.001s
I20250628 07:52:06.094134   708 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.129:34311
I20250628 07:52:06.094275   869 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.129:34311 every 8 connection(s)
I20250628 07:52:06.097556   708 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
I20250628 07:52:06.098672   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 708
I20250628 07:52:06.099172   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/instance
I20250628 07:52:06.112412   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.130:0
--local_ip_for_outbound_sockets=127.0.107.130
--webserver_interface=127.0.107.130
--webserver_port=0
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:52:06.192371   874 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:06.193459   874 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:06.194690   874 heartbeater.cc:507] Master 127.0.107.190:37209 requested a full tablet report, sending...
I20250628 07:52:06.198758   472 ts_manager.cc:194] Registered new tserver with Master: 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:06.203042   873 heartbeater.cc:344] Connected to a master server at 127.0.107.189:33023
I20250628 07:52:06.203399   873 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:06.204207   873 heartbeater.cc:507] Master 127.0.107.189:33023 requested a full tablet report, sending...
I20250628 07:52:06.207346   547 ts_manager.cc:194] Registered new tserver with Master: 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:06.211737   870 heartbeater.cc:344] Connected to a master server at 127.0.107.188:42677
I20250628 07:52:06.212111   870 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:06.212991   870 heartbeater.cc:507] Master 127.0.107.188:42677 requested a full tablet report, sending...
I20250628 07:52:06.216429   626 ts_manager.cc:194] Registered new tserver with Master: 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:06.220149   626 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.129:52405
I20250628 07:52:06.369760   881 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:06.371192   882 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:06.371723   882 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:06.378621   881 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:06.383222   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:06.383800   555 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5fd67221fa5f424ea0f83e582629d276 in term 1.
I20250628 07:52:06.384968   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 
I20250628 07:52:06.398684   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:06.400851   881 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:06.402004   882 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:06.404390   881 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:52:06.406445   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:06.408456   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:06.409940   481 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 023ed4cb582a44509bccf42b52c494d7 in term 1.
I20250628 07:52:06.411108   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:06.412439   882 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:52:06.412838   882 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:06.413233   881 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 992ac4c27fb744deb1edf7765553ef08)
I20250628 07:52:06.413659   881 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:06.414053   882 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:06.428458   881 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:06.433001   481 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Rejecting Update request from peer 992ac4c27fb744deb1edf7765553ef08 for earlier term 1. Current term is 2. Ops: []
I20250628 07:52:06.433882   881 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 election: Requested vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:06.435349   882 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:06.437240   882 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:06.437997   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7"
I20250628 07:52:06.438508   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08"
I20250628 07:52:06.438767   555 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 5fd67221fa5f424ea0f83e582629d276 in current term 2: Already voted for candidate 023ed4cb582a44509bccf42b52c494d7 in this term.
I20250628 07:52:06.440512   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:06.441258   482 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 in current term 2: Already voted for candidate 5fd67221fa5f424ea0f83e582629d276 in this term.
I20250628 07:52:06.441882   883 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 }, Status: INVALID_TERM, Last received: 1.4, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20250628 07:52:06.443785   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:06.445039   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08"
I20250628 07:52:06.446913   555 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Rejecting Update request from peer 992ac4c27fb744deb1edf7765553ef08 for earlier term 1. Current term is 2. Ops: []
I20250628 07:52:06.451360   881 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
I20250628 07:52:06.453891   886 raft_consensus.cc:3053] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 LEADER]: Stepping down as leader of term 1
I20250628 07:52:06.454334   886 raft_consensus.cc:738] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 992ac4c27fb744deb1edf7765553ef08, State: Running, Role: LEADER
I20250628 07:52:06.455051   534 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7; no voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:06.454464   885 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 }, Status: INVALID_TERM, Last received: 1.4, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20250628 07:52:06.456066   882 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
I20250628 07:52:06.456568   886 consensus_queue.cc:260] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:06.457760   886 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 1 FOLLOWER]: Advancing to term 2
W20250628 07:52:06.582909   877 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:52:06.583431   877 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:52:06.584038   877 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:52:06.616614   877 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:52:06.617710   877 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.130
I20250628 07:52:06.652465   877 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.0.107.130
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.130
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:52:06.653834   877 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:52:06.655376   877 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:52:06.672930   893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:06.724715   870 heartbeater.cc:499] Master 127.0.107.188:42677 was elected leader, sending a full tablet report...
I20250628 07:52:07.050462   900 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:07.058982   900 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:07.095110   439 master_runner.cc:263] No leader master found from master 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:07.095453   439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:07.095765   697 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:52:07.107347   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:07.108207   481 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 2.
I20250628 07:52:07.110015   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08; no voters: 
I20250628 07:52:07.111785   901 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:07.113145   901 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:07.117131   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:07.119370   900 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:07.120338   900 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 2 FOLLOWER]: Leader pre-election won for term 3
I20250628 07:52:07.120918   555 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 2.
I20250628 07:52:07.121145   900 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:07.121451   900 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:52:07.124373   901 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:07.125262   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:07.125823   481 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 023ed4cb582a44509bccf42b52c494d7 in term 2.
I20250628 07:52:07.126879   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 
I20250628 07:52:07.127753   901 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Leader pre-election won for term 3
I20250628 07:52:07.128087   901 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:07.128417   901 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:52:07.138314   901 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:07.139528   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:07.142385   584 master_runner.cc:232] Existing masters have differing terms: 2 vs 3
I20250628 07:52:07.142777   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:07.152850   901 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 3 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:07.153470   481 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:07.154145   481 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:52:07.158254   635 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08"
I20250628 07:52:07.169530   481 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 023ed4cb582a44509bccf42b52c494d7 in term 3.
I20250628 07:52:07.168807   900 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:07.175457   636 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 in current term 3: Already voted for candidate 992ac4c27fb744deb1edf7765553ef08 in this term.
I20250628 07:52:07.180516   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:07.181795   901 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 FOLLOWER]: Leader election won for term 3
I20250628 07:52:07.184230   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:07.185156   482 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 992ac4c27fb744deb1edf7765553ef08 in current term 3: Already voted for candidate 023ed4cb582a44509bccf42b52c494d7 in this term.
I20250628 07:52:07.188270   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7"
I20250628 07:52:07.189374   900 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 3 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:07.193204   901 raft_consensus.cc:695] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 LEADER]: Becoming Leader. State: Replica: 023ed4cb582a44509bccf42b52c494d7, State: Running, Role: LEADER
I20250628 07:52:07.194181   901 consensus_queue.cc:237] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:07.209544   903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.210511   903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:07.213487   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:07.215102   900 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
I20250628 07:52:07.230921   905 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:52:07.248677   905 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:52:07.257385   905 catalog_manager.cc:1261] Loaded cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:07.263710   905 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:52:07.271701   905 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:52:07.287645   905 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Loaded TSK: 0
I20250628 07:52:07.305243   905 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:52:07.587077   481 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Refusing update from remote peer 023ed4cb582a44509bccf42b52c494d7: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20250628 07:52:07.589761   903 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20250628 07:52:07.615073   906 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.616168   906 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:07.655191   906 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.656184   906 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:07.662385   636 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Refusing update from remote peer 023ed4cb582a44509bccf42b52c494d7: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20250628 07:52:07.662228   901 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.663190   901 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:07.664407   903 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20250628 07:52:07.694034   900 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.695104   900 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:07.700084   912 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:07.701048   912 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: FOLLOWER
W20250628 07:52:08.070945   892 debug-util.cc:398] Leaking SignalData structure 0x7b08000186a0 after lost signal to thread 877
W20250628 07:52:08.081099   877 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.413s	user 0.401s	sys 0.793s
W20250628 07:52:08.083104   896 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:52:08.083500   895 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1412 milliseconds
W20250628 07:52:08.083925   877 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.415s	user 0.402s	sys 0.793s
I20250628 07:52:08.084288   877 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
W20250628 07:52:06.673619   894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:08.090260   877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:52:08.092881   877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:52:08.094331   877 hybrid_clock.cc:648] HybridClock initialized: now 1751097128094294 us; error 55 us; skew 500 ppm
I20250628 07:52:08.095638   877 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:52:08.096001   903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:08.096645   903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:08.106936   877 webserver.cc:469] Webserver started at http://127.0.107.130:33155/ using document root <none> and password file <none>
I20250628 07:52:08.107816   877 fs_manager.cc:362] Metadata directory not provided
I20250628 07:52:08.108001   877 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:52:08.108433   877 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:52:08.112897   877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data/instance:
uuid: "82f43cdfab544b17a6a71033983e65d4"
format_stamp: "Formatted at 2025-06-28 07:52:08 on dist-test-slave-kfnw"
I20250628 07:52:08.114010   877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/instance:
uuid: "82f43cdfab544b17a6a71033983e65d4"
format_stamp: "Formatted at 2025-06-28 07:52:08 on dist-test-slave-kfnw"
I20250628 07:52:08.120965   877 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
I20250628 07:52:08.124604   906 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:08.125079   906 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:08.126561   906 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:08.127388   918 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:08.127579   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:08.127652   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:08.128744   877 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
I20250628 07:52:08.129042   877 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal
uuid: "82f43cdfab544b17a6a71033983e65d4"
format_stamp: "Formatted at 2025-06-28 07:52:08 on dist-test-slave-kfnw"
I20250628 07:52:08.129503   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:08.129382   877 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:08.130132   906 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:08.178893   877 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:08.180665   877 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:08.181068   877 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:08.183334   877 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:52:08.187582   877 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:52:08.187789   877 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:08.188033   877 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:52:08.188187   877 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:08.223529   873 heartbeater.cc:499] Master 127.0.107.189:33023 was elected leader, sending a full tablet report...
I20250628 07:52:08.341439   877 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.130:35061
I20250628 07:52:08.341488  1030 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.130:35061 every 8 connection(s)
I20250628 07:52:08.344645   877 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
I20250628 07:52:08.355135   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 877
I20250628 07:52:08.355773   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/instance
I20250628 07:52:08.378754   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.131:0
--local_ip_for_outbound_sockets=127.0.107.131
--webserver_interface=127.0.107.131
--webserver_port=0
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:52:08.390619  1033 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:08.391228  1033 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:08.392042  1031 heartbeater.cc:344] Connected to a master server at 127.0.107.188:42677
I20250628 07:52:08.392395  1031 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:08.392495  1033 heartbeater.cc:507] Master 127.0.107.190:37209 requested a full tablet report, sending...
I20250628 07:52:08.393280  1032 heartbeater.cc:344] Connected to a master server at 127.0.107.189:33023
I20250628 07:52:08.393316  1031 heartbeater.cc:507] Master 127.0.107.188:42677 requested a full tablet report, sending...
I20250628 07:52:08.393626  1032 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:08.395135  1032 heartbeater.cc:507] Master 127.0.107.189:33023 requested a full tablet report, sending...
I20250628 07:52:08.395968   472 ts_manager.cc:194] Registered new tserver with Master: 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:08.397176   547 ts_manager.cc:194] Registered new tserver with Master: 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:08.397783   626 ts_manager.cc:194] Registered new tserver with Master: 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:08.399544   547 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.130:44463
W20250628 07:52:08.688238  1041 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:52:08.688737  1041 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:52:08.689707  1041 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:52:08.721184  1041 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:52:08.722226  1041 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.131
I20250628 07:52:08.758194  1041 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.0.107.131
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.131
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:52:08.759475  1041 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:52:08.761003  1041 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:52:08.778105  1051 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:08.902985  1032 heartbeater.cc:499] Master 127.0.107.189:33023 was elected leader, sending a full tablet report...
I20250628 07:52:09.058552  1054 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:09.058964  1054 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:09.060633  1054 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:09.061196   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:09.062278   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:09.064358   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:09.064913  1054 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:09.569957  1055 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:09.570348  1055 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:09.571844  1055 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:09.572708   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:09.573539   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:09.575455   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:09.576459  1055 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:52:08.778630  1049 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:52:08.778239  1048 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:52:09.970069  1050 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20250628 07:52:09.970187  1041 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:52:09.973963  1041 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:52:09.976176  1041 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:52:09.977519  1041 hybrid_clock.cc:648] HybridClock initialized: now 1751097129977479 us; error 58 us; skew 500 ppm
I20250628 07:52:09.978303  1041 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:52:09.985819  1041 webserver.cc:469] Webserver started at http://127.0.107.131:38667/ using document root <none> and password file <none>
I20250628 07:52:09.986758  1041 fs_manager.cc:362] Metadata directory not provided
I20250628 07:52:09.987015  1041 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:52:09.987485  1041 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:52:09.991968  1041 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data/instance:
uuid: "6a58514e01ea4989809bc2992fb2b806"
format_stamp: "Formatted at 2025-06-28 07:52:09 on dist-test-slave-kfnw"
I20250628 07:52:09.993587  1041 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/instance:
uuid: "6a58514e01ea4989809bc2992fb2b806"
format_stamp: "Formatted at 2025-06-28 07:52:09 on dist-test-slave-kfnw"
I20250628 07:52:10.000919  1041 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.005s
I20250628 07:52:10.006592  1060 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:10.007597  1041 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
I20250628 07:52:10.007954  1041 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal
uuid: "6a58514e01ea4989809bc2992fb2b806"
format_stamp: "Formatted at 2025-06-28 07:52:09 on dist-test-slave-kfnw"
I20250628 07:52:10.008263  1041 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:10.072777  1041 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:10.074713  1041 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:10.075148  1041 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:10.077641  1041 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:52:10.078966  1066 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:10.079495  1066 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:10.081367  1066 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:10.082196   482 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:10.083894   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:10.085795  1041 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:52:10.085829   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:10.086107  1041 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
I20250628 07:52:10.086463  1041 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:52:10.086730  1041 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:10.086715  1066 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:10.246528  1041 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.131:37709
I20250628 07:52:10.246628  1173 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.131:37709 every 8 connection(s)
I20250628 07:52:10.248960  1041 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
I20250628 07:52:10.251327   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 1041
I20250628 07:52:10.251842   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/instance
I20250628 07:52:10.290853  1175 heartbeater.cc:344] Connected to a master server at 127.0.107.189:33023
I20250628 07:52:10.291402  1174 heartbeater.cc:344] Connected to a master server at 127.0.107.188:42677
I20250628 07:52:10.291455  1175 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:10.291816  1174 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:10.290961  1176 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:10.292802  1176 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:10.292769  1175 heartbeater.cc:507] Master 127.0.107.189:33023 requested a full tablet report, sending...
I20250628 07:52:10.294237  1176 heartbeater.cc:507] Master 127.0.107.190:37209 requested a full tablet report, sending...
I20250628 07:52:10.296293   547 ts_manager.cc:194] Registered new tserver with Master: 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:10.296785   472 ts_manager.cc:194] Registered new tserver with Master: 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:10.293016  1174 heartbeater.cc:507] Master 127.0.107.188:42677 requested a full tablet report, sending...
I20250628 07:52:10.297672   547 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.131:60285
I20250628 07:52:10.299875   625 ts_manager.cc:194] Registered new tserver with Master: 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:10.319501   430 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
I20250628 07:52:10.376718   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e15cc9e8d1654a9792589bcb049629d7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.423846   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e15cc9e8d1654a9792589bcb049629d7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.377066   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d514003edaad41f1a8cc2a23304a9529"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.427693   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d514003edaad41f1a8cc2a23304a9529 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.380592   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bb992e867b3d4bc7b89ea89a5a2c414e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.430526   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bb992e867b3d4bc7b89ea89a5a2c414e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.376672   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8a3c08497b904f24a4dbaa713e7c9ab7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.451112   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8a3c08497b904f24a4dbaa713e7c9ab7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.463686  1207 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:10.464286  1207 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:10.471009   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:10.475342  1207 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:10.475006   635 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:10.477332   459 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:10.481926  1207 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:10.482575   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 439
I20250628 07:52:10.515358   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_e15cc9e8d1654a9792589bcb049629d7" } modify_external_catalogs: true
I20250628 07:52:10.516016   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_e15cc9e8d1654a9792589bcb049629d7" } modify_external_catalogs: true
I20250628 07:52:10.516615   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d514003edaad41f1a8cc2a23304a9529" } new_table_name: "default.Table_3bd28637c21b4d4881c1880f63a56b85" modify_external_catalogs: true
I20250628 07:52:10.518229   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_bb992e867b3d4bc7b89ea89a5a2c414e" } new_table_name: "default.Table_5c45cd3ccc2c448dbf251659f548a345" modify_external_catalogs: true
W20250628 07:52:10.524060   531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.0.107.190:37209 (error 108) [suppressed 6 similar messages]
I20250628 07:52:10.543298   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8a3c08497b904f24a4dbaa713e7c9ab7" } modify_external_catalogs: true
I20250628 07:52:10.543823   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8a3c08497b904f24a4dbaa713e7c9ab7" } modify_external_catalogs: true
W20250628 07:52:10.555802   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:10.568001   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 93d993bfdab94f9b8803a5901b1df769 consensus state has no leader
I20250628 07:52:10.579094   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 3fa9add48f5446f0b69a5890f228912a
W20250628 07:52:10.593577   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1cdae9cac7364880a715726890f23485 has no consensus state
I20250628 07:52:10.595954  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 93d993bfdab94f9b8803a5901b1df769 (DEFAULT_TABLE table=default.Table_bb992e867b3d4bc7b89ea89a5a2c414e [id=4a76c0057963463f84ebb91d85f9725e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.595947   966 tablet_service.cc:1468] Processing CreateTablet for tablet 93d993bfdab94f9b8803a5901b1df769 (DEFAULT_TABLE table=default.Table_bb992e867b3d4bc7b89ea89a5a2c414e [id=4a76c0057963463f84ebb91d85f9725e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.598001   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 93d993bfdab94f9b8803a5901b1df769. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:10.600940   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_5c45cd3ccc2c448dbf251659f548a345 [id=4a76c0057963463f84ebb91d85f9725e], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 93d993bfdab94f9b8803a5901b1df769 consensus state has no leader
I20250628 07:52:10.602129  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 93d993bfdab94f9b8803a5901b1df769. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.606825   546 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet acd5538c446748289b72d7f0e96e9d7e
I20250628 07:52:10.617506   805 tablet_service.cc:1468] Processing CreateTablet for tablet 93d993bfdab94f9b8803a5901b1df769 (DEFAULT_TABLE table=default.Table_bb992e867b3d4bc7b89ea89a5a2c414e [id=4a76c0057963463f84ebb91d85f9725e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.619557   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 93d993bfdab94f9b8803a5901b1df769. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.646068   965 tablet_service.cc:1468] Processing CreateTablet for tablet 1cdae9cac7364880a715726890f23485 (DEFAULT_TABLE table=default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.646927  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 1cdae9cac7364880a715726890f23485 (DEFAULT_TABLE table=default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.647271   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cdae9cac7364880a715726890f23485. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.648139  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cdae9cac7364880a715726890f23485. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.649305   804 tablet_service.cc:1468] Processing CreateTablet for tablet 1cdae9cac7364880a715726890f23485 (DEFAULT_TABLE table=default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.650493   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cdae9cac7364880a715726890f23485. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.653887  1229 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:10.662168  1229 tablet_bootstrap.cc:654] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.664271  1230 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:10.664588  1229 log.cc:826] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:10.673944  1230 tablet_bootstrap.cc:654] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.677431  1230 log.cc:826] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:10.691249  1230 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:10.692169  1230 ts_tablet_manager.cc:1397] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.030s	user 0.026s	sys 0.000s
I20250628 07:52:10.701117   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4b0c3fc133ca410ba9b6151c429caeef"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.702908   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4b0c3fc133ca410ba9b6151c429caeef in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.712769   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5d03e239a7ab48779e36a906630751bd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.714537   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5d03e239a7ab48779e36a906630751bd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.719084   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_41a1bc0dc63041d1829de3ad0475bd56"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.720726   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_41a1bc0dc63041d1829de3ad0475bd56 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.733783   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4d1ddae3cd774da5bce99ead9d0d7405"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.735817   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4d1ddae3cd774da5bce99ead9d0d7405 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.757653  1229 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:10.759055  1229 ts_tablet_manager.cc:1397] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.106s	user 0.033s	sys 0.034s
I20250628 07:52:10.759986  1230 raft_consensus.cc:357] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.762864  1230 raft_consensus.cc:383] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:10.763700  1230 raft_consensus.cc:738] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:10.765496  1230 consensus_queue.cc:260] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:10.769767   874 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:37209 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111)
I20250628 07:52:10.788291  1175 heartbeater.cc:499] Master 127.0.107.189:33023 was elected leader, sending a full tablet report...
I20250628 07:52:10.789659  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 752fc37e7210458fb0c631d01204a3bb (DEFAULT_TABLE table=default.Table_4b0c3fc133ca410ba9b6151c429caeef [id=4c6801aa5535410980af48c4ca4eb5d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.790756  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 752fc37e7210458fb0c631d01204a3bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.795220   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_3bd28637c21b4d4881c1880f63a56b85" } modify_external_catalogs: true
I20250628 07:52:10.795534   804 tablet_service.cc:1468] Processing CreateTablet for tablet 752fc37e7210458fb0c631d01204a3bb (DEFAULT_TABLE table=default.Table_4b0c3fc133ca410ba9b6151c429caeef [id=4c6801aa5535410980af48c4ca4eb5d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.795645   965 tablet_service.cc:1468] Processing CreateTablet for tablet 752fc37e7210458fb0c631d01204a3bb (DEFAULT_TABLE table=default.Table_4b0c3fc133ca410ba9b6151c429caeef [id=4c6801aa5535410980af48c4ca4eb5d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.795754   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_3bd28637c21b4d4881c1880f63a56b85" } modify_external_catalogs: true
I20250628 07:52:10.796630   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 752fc37e7210458fb0c631d01204a3bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.796736   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 752fc37e7210458fb0c631d01204a3bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.797348   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4b0c3fc133ca410ba9b6151c429caeef" } new_table_name: "default.Table_9c39cbe761084240846a9147b9666d33" modify_external_catalogs: true
W20250628 07:52:10.802129  1176 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:37209 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111)
I20250628 07:52:10.807071  1230 ts_tablet_manager.cc:1428] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.114s	user 0.027s	sys 0.016s
I20250628 07:52:10.808063  1230 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:10.809334   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5c45cd3ccc2c448dbf251659f548a345" } new_table_name: "default.Table_ac00834b064d4a89ba7c910f007cef1e" modify_external_catalogs: true
I20250628 07:52:10.815919  1230 tablet_bootstrap.cc:654] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.816512   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_41a1bc0dc63041d1829de3ad0475bd56" } modify_external_catalogs: true
I20250628 07:52:10.817039   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_41a1bc0dc63041d1829de3ad0475bd56" } modify_external_catalogs: true
I20250628 07:52:10.834650   803 tablet_service.cc:1468] Processing CreateTablet for tablet b5f50ad0815b433287d36bb209bee50d (DEFAULT_TABLE table=default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.835263   965 tablet_service.cc:1468] Processing CreateTablet for tablet b5f50ad0815b433287d36bb209bee50d (DEFAULT_TABLE table=default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.835892   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b5f50ad0815b433287d36bb209bee50d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.836315   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b5f50ad0815b433287d36bb209bee50d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.839802  1109 tablet_service.cc:1468] Processing CreateTablet for tablet b5f50ad0815b433287d36bb209bee50d (DEFAULT_TABLE table=default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.840996  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b5f50ad0815b433287d36bb209bee50d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.841921  1229 raft_consensus.cc:357] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.842850  1229 raft_consensus.cc:383] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:10.843828  1229 raft_consensus.cc:738] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:10.844795  1229 consensus_queue.cc:260] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.846441  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 80f812df4f15406ebe0311c19bb654a7 (DEFAULT_TABLE table=default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.847952  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80f812df4f15406ebe0311c19bb654a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.855158  1229 ts_tablet_manager.cc:1428] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.095s	user 0.037s	sys 0.003s
I20250628 07:52:10.863539   965 tablet_service.cc:1468] Processing CreateTablet for tablet 80f812df4f15406ebe0311c19bb654a7 (DEFAULT_TABLE table=default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.864552   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80f812df4f15406ebe0311c19bb654a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.866084   804 tablet_service.cc:1468] Processing CreateTablet for tablet 80f812df4f15406ebe0311c19bb654a7 (DEFAULT_TABLE table=default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:10.867322   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80f812df4f15406ebe0311c19bb654a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:10.867672  1229 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:10.867854   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_5c45cd3ccc2c448dbf251659f548a345 [id=4a76c0057963463f84ebb91d85f9725e]). Expected version 1 got 0
W20250628 07:52:10.875628  1036 tablet.cc:2378] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:10.886744  1229 tablet_bootstrap.cc:654] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.898721  1229 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:10.900990  1229 ts_tablet_manager.cc:1397] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.034s	user 0.007s	sys 0.005s
I20250628 07:52:10.906481  1229 raft_consensus.cc:357] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.909919  1229 raft_consensus.cc:383] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:10.910712  1229 raft_consensus.cc:738] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:10.912230  1229 consensus_queue.cc:260] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.917946  1229 ts_tablet_manager.cc:1428] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.001s
I20250628 07:52:10.920511  1229 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:10.928591   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 752fc37e7210458fb0c631d01204a3bb consensus state has no leader
I20250628 07:52:10.933197  1229 tablet_bootstrap.cc:654] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.947316  1229 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:10.949221  1229 ts_tablet_manager.cc:1397] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.029s	user 0.012s	sys 0.000s
I20250628 07:52:10.953645  1229 raft_consensus.cc:357] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.957166  1229 raft_consensus.cc:383] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:10.957839  1229 raft_consensus.cc:738] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:10.958786   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_2770a34952aa436fb1566f2efc41b514"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:10.958840  1229 consensus_queue.cc:260] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:10.959230   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6e7d61663821420fb5bf9aa11076b8b1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:10.960757   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_2770a34952aa436fb1566f2efc41b514 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:10.960788   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6e7d61663821420fb5bf9aa11076b8b1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.963958   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_62f8e336893a4e0b8e2f21306fa953b8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:10.965206  1229 ts_tablet_manager.cc:1428] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.015s	user 0.007s	sys 0.000s
W20250628 07:52:10.965708   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_62f8e336893a4e0b8e2f21306fa953b8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:10.966089  1229 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:10.973711  1222 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:10.976051  1222 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:10.978391   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:10.981688   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:10.982741   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:10.983274  1229 tablet_bootstrap.cc:654] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:10.986915  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.002645  1234 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:11.003921   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 80f812df4f15406ebe0311c19bb654a7 because the tablet was not found. No further retry: Not found: Tablet not found: 80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:11.011127  1229 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
W20250628 07:52:11.011678   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 80f812df4f15406ebe0311c19bb654a7 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 80f812df4f15406ebe0311c19bb654a7 already in progress: creating tablet
I20250628 07:52:11.012483  1229 ts_tablet_manager.cc:1397] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.047s	user 0.011s	sys 0.001s
I20250628 07:52:11.015916  1229 raft_consensus.cc:357] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.019811  1229 raft_consensus.cc:383] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.020506  1229 raft_consensus.cc:738] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.021692  1229 consensus_queue.cc:260] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:11.023217  1181 tablet.cc:2378] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:11.026480   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8807b0cad1f9413aaa097f7c6158643a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:11.028870   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_5c45cd3ccc2c448dbf251659f548a345 [id=4a76c0057963463f84ebb91d85f9725e]). Expected version 1 got 0
I20250628 07:52:11.031006   805 tablet_service.cc:1468] Processing CreateTablet for tablet 63348a64bb7044dbad9065afff179fbc (DEFAULT_TABLE table=default.Table_4d1ddae3cd774da5bce99ead9d0d7405 [id=ff9dc70424f54129963f83df1d05463f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.031502   966 tablet_service.cc:1468] Processing CreateTablet for tablet 63348a64bb7044dbad9065afff179fbc (DEFAULT_TABLE table=default.Table_4d1ddae3cd774da5bce99ead9d0d7405 [id=ff9dc70424f54129963f83df1d05463f]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:11.032155   543 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 93d993bfdab94f9b8803a5901b1df769 consensus state has no leader
I20250628 07:52:11.032177   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63348a64bb7044dbad9065afff179fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.032564   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63348a64bb7044dbad9065afff179fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.033221   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1cdae9cac7364880a715726890f23485
I20250628 07:52:11.035465  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 63348a64bb7044dbad9065afff179fbc (DEFAULT_TABLE table=default.Table_4d1ddae3cd774da5bce99ead9d0d7405 [id=ff9dc70424f54129963f83df1d05463f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.036552  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63348a64bb7044dbad9065afff179fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.038542  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.039201   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:11.040072   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:11.041764  1229 ts_tablet_manager.cc:1428] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.028s	user 0.004s	sys 0.002s
W20250628 07:52:11.043488   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1cdae9cac7364880a715726890f23485 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1cdae9cac7364880a715726890f23485 already in progress: creating tablet
W20250628 07:52:11.044595   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 1cdae9cac7364880a715726890f23485 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1cdae9cac7364880a715726890f23485 already in progress: creating tablet
W20250628 07:52:11.029196   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8807b0cad1f9413aaa097f7c6158643a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.066429  1237 raft_consensus.cc:491] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:11.066985  1229 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:11.067025  1237 raft_consensus.cc:513] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.070092   804 tablet_service.cc:1468] Processing CreateTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (DEFAULT_TABLE table=default.Table_6e7d61663821420fb5bf9aa11076b8b1 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.071185   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6e912d18dcff4a938a867ea06ee7e0f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.072911   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]). Expected version 1 got 0
I20250628 07:52:11.075168  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (DEFAULT_TABLE table=default.Table_6e7d61663821420fb5bf9aa11076b8b1 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.076251  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6e912d18dcff4a938a867ea06ee7e0f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.078027  1105 tablet_service.cc:1468] Processing CreateTablet for tablet fbd97bd5efec4a989374c6c08e312e08 (DEFAULT_TABLE table=default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.079353  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbd97bd5efec4a989374c6c08e312e08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.086077  1237 leader_election.cc:290] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:11.086529  1229 tablet_bootstrap.cc:654] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.092031   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_ac00834b064d4a89ba7c910f007cef1e [id=4a76c0057963463f84ebb91d85f9725e]). Expected version 2 got 0
I20250628 07:52:11.099946  1235 raft_consensus.cc:491] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:11.100662  1235 raft_consensus.cc:513] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.108655  1235 leader_election.cc:290] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:11.109354   965 tablet_service.cc:1468] Processing CreateTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (DEFAULT_TABLE table=default.Table_62f8e336893a4e0b8e2f21306fa953b8 [id=0678e3fca14f4accbcf09e6ff76db055]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.110921   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.119163   964 tablet_service.cc:1468] Processing CreateTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (DEFAULT_TABLE table=default.Table_6e7d61663821420fb5bf9aa11076b8b1 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.120301   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6e912d18dcff4a938a867ea06ee7e0f0. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:11.121452  1033 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:37209 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111)
I20250628 07:52:11.135052   803 tablet_service.cc:1468] Processing CreateTablet for tablet fbd97bd5efec4a989374c6c08e312e08 (DEFAULT_TABLE table=default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.136112   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbd97bd5efec4a989374c6c08e312e08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.136098  1245 tablet_replica.cc:331] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:11.136967  1245 raft_consensus.cc:2241] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.137571  1245 raft_consensus.cc:2270] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:11.146477   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 80f812df4f15406ebe0311c19bb654a7 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 80f812df4f15406ebe0311c19bb654a7 already in progress: creating tablet
I20250628 07:52:11.146553   966 tablet_service.cc:1468] Processing CreateTablet for tablet fbd97bd5efec4a989374c6c08e312e08 (DEFAULT_TABLE table=default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.147173   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4d1ddae3cd774da5bce99ead9d0d7405" } new_table_name: "default.Table_900934bdf61248cca0ef6d9dc5e08359" modify_external_catalogs: true
I20250628 07:52:11.147658   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbd97bd5efec4a989374c6c08e312e08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.149333   805 tablet_service.cc:1468] Processing CreateTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (DEFAULT_TABLE table=default.Table_62f8e336893a4e0b8e2f21306fa953b8 [id=0678e3fca14f4accbcf09e6ff76db055]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.150359   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.158756   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:11.159463   986 raft_consensus.cc:2466] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:11.160966  1062 leader_election.cc:304] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:11.161960  1235 raft_consensus.cc:2802] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:11.162334  1235 raft_consensus.cc:491] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:11.162686  1235 raft_consensus.cc:3058] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:11.167228  1230 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:11.167685  1230 ts_tablet_manager.cc:1397] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.360s	user 0.028s	sys 0.049s
I20250628 07:52:11.169575  1235 raft_consensus.cc:513] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.170385  1230 raft_consensus.cc:357] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.171202  1230 raft_consensus.cc:383] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.171530  1230 raft_consensus.cc:738] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.172175   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:11.172207  1230 consensus_queue.cc:260] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.172753   986 raft_consensus.cc:3058] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:11.179306   986 raft_consensus.cc:2466] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:11.180207  1062 leader_election.cc:304] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:11.184648  1245 ts_tablet_manager.cc:1905] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.189110   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5d03e239a7ab48779e36a906630751bd" } modify_external_catalogs: true
I20250628 07:52:11.192349  1235 leader_election.cc:290] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:11.198382   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5d03e239a7ab48779e36a906630751bd" } modify_external_catalogs: true
I20250628 07:52:11.198128  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (DEFAULT_TABLE table=default.Table_62f8e336893a4e0b8e2f21306fa953b8 [id=0678e3fca14f4accbcf09e6ff76db055]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.199548  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:11.205612   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 63348a64bb7044dbad9065afff179fbc consensus state has no leader
I20250628 07:52:11.212106  1255 raft_consensus.cc:2802] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:11.212633  1255 raft_consensus.cc:695] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:11.213584  1255 consensus_queue.cc:237] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.225131  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.225250   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_448f6b4c3c354e76aaf65492579adba1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.226745   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1cdae9cac7364880a715726890f23485 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1cdae9cac7364880a715726890f23485 already in progress: creating tablet
W20250628 07:52:11.227200   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_448f6b4c3c354e76aaf65492579adba1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.230232   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_229010e2d61b49ee9e17f2d21ad323c6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.231910   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_229010e2d61b49ee9e17f2d21ad323c6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.234921   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_0ef22eef86e5432bb09a3f878ed94458"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.236565   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0ef22eef86e5432bb09a3f878ed94458 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.238286  1234 tablet_bootstrap.cc:654] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.241220   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ac00834b064d4a89ba7c910f007cef1e" } new_table_name: "default.Table_8d2b2eef04ee43419aa614b4174b338e" modify_external_catalogs: true
I20250628 07:52:11.247118   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8807b0cad1f9413aaa097f7c6158643a" } modify_external_catalogs: true
I20250628 07:52:11.263031   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8807b0cad1f9413aaa097f7c6158643a" } modify_external_catalogs: true
I20250628 07:52:11.263299   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet b5f50ad0815b433287d36bb209bee50d
I20250628 07:52:11.266484  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet b5f50ad0815b433287d36bb209bee50d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:11.269928   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b5f50ad0815b433287d36bb209bee50d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b5f50ad0815b433287d36bb209bee50d already in progress: creating tablet
I20250628 07:52:11.270560   805 tablet_service.cc:1515] Processing DeleteTablet for tablet b5f50ad0815b433287d36bb209bee50d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:11.273816   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b5f50ad0815b433287d36bb209bee50d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b5f50ad0815b433287d36bb209bee50d already in progress: creating tablet
I20250628 07:52:11.256507   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:11.275789  1245 ts_tablet_manager.cc:1918] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:11.268908   965 tablet_service.cc:1515] Processing DeleteTablet for tablet b5f50ad0815b433287d36bb209bee50d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:11.276547   920 leader_election.cc:343] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.277519  1245 log.cc:1199] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/1cdae9cac7364880a715726890f23485
I20250628 07:52:11.278710  1245 ts_tablet_manager.cc:1939] T 1cdae9cac7364880a715726890f23485 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:11.282238   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 1cdae9cac7364880a715726890f23485 (table default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]) successfully deleted
W20250628 07:52:11.283684   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 93d993bfdab94f9b8803a5901b1df769 consensus state has no leader
I20250628 07:52:11.285501  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:11.287212  1245 tablet_replica.cc:331] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:11.287632   921 leader_election.cc:304] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:11.288677  1237 raft_consensus.cc:2747] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:11.289274  1245 raft_consensus.cc:2241] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.290177  1245 raft_consensus.cc:2270] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:11.290159  1234 log.cc:826] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:11.292850  1245 ts_tablet_manager.cc:1905] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.296517   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:11.296099   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "93d993bfdab94f9b8803a5901b1df769" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:11.297880  1062 leader_election.cc:343] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:11.299201  1062 leader_election.cc:343] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.302018  1229 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:11.302438  1229 ts_tablet_manager.cc:1397] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.236s	user 0.013s	sys 0.007s
I20250628 07:52:11.306370  1229 raft_consensus.cc:357] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.307054  1229 raft_consensus.cc:383] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.307327  1229 raft_consensus.cc:738] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.308072  1229 consensus_queue.cc:260] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.320586   540 catalog_manager.cc:5582] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } }
I20250628 07:52:11.322599   805 tablet_service.cc:1468] Processing CreateTablet for tablet c1a06bbe216f419598a587056f9b8d05 (DEFAULT_TABLE table=default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.323683   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c1a06bbe216f419598a587056f9b8d05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.326200  1108 tablet_service.cc:1468] Processing CreateTablet for tablet c1a06bbe216f419598a587056f9b8d05 (DEFAULT_TABLE table=default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.327263  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c1a06bbe216f419598a587056f9b8d05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.328212   966 tablet_service.cc:1468] Processing CreateTablet for tablet c1a06bbe216f419598a587056f9b8d05 (DEFAULT_TABLE table=default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.329257   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c1a06bbe216f419598a587056f9b8d05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.329223   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_8d2b2eef04ee43419aa614b4174b338e [id=4a76c0057963463f84ebb91d85f9725e]). Expected version 3 got 0
I20250628 07:52:11.344172  1234 tablet_bootstrap.cc:492] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:11.346207  1234 ts_tablet_manager.cc:1397] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.344s	user 0.016s	sys 0.000s
I20250628 07:52:11.346848  1230 ts_tablet_manager.cc:1428] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.179s	user 0.018s	sys 0.047s
I20250628 07:52:11.347939  1230 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:11.354810  1245 ts_tablet_manager.cc:1918] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:11.355280  1245 log.cc:1199] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/b5f50ad0815b433287d36bb209bee50d
I20250628 07:52:11.355971  1245 ts_tablet_manager.cc:1939] T b5f50ad0815b433287d36bb209bee50d P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:11.358439   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet b5f50ad0815b433287d36bb209bee50d (table default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]) successfully deleted
I20250628 07:52:11.358520  1229 ts_tablet_manager.cc:1428] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.056s	user 0.006s	sys 0.001s
I20250628 07:52:11.359403   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet c1a06bbe216f419598a587056f9b8d05
I20250628 07:52:11.361047   805 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:11.363087  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.363157   965 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:11.364060   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet c1a06bbe216f419598a587056f9b8d05 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c1a06bbe216f419598a587056f9b8d05 already in progress: creating tablet
W20250628 07:52:11.364351   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c1a06bbe216f419598a587056f9b8d05 because the tablet was not found. No further retry: Not found: Tablet not found: c1a06bbe216f419598a587056f9b8d05
W20250628 07:52:11.365442   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet c1a06bbe216f419598a587056f9b8d05 because the tablet was not found. No further retry: Not found: Tablet not found: c1a06bbe216f419598a587056f9b8d05
I20250628 07:52:11.367851   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:11.368242  1230 tablet_bootstrap.cc:654] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.368575  1245 tablet_replica.cc:331] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:11.369316  1245 raft_consensus.cc:2241] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.369798  1245 raft_consensus.cc:2270] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:11.371526  1245 ts_tablet_manager.cc:1905] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.383824  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.384586  1246 tablet_replica.cc:331] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:11.385382  1246 raft_consensus.cc:2241] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.385854  1246 raft_consensus.cc:2270] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:11.398128   805 tablet_service.cc:1468] Processing CreateTablet for tablet b8b1e09d0f224c10ab8df26f12239254 (DEFAULT_TABLE table=default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.399201   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b8b1e09d0f224c10ab8df26f12239254. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.399468  1105 tablet_service.cc:1468] Processing CreateTablet for tablet b8b1e09d0f224c10ab8df26f12239254 (DEFAULT_TABLE table=default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.400519  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b8b1e09d0f224c10ab8df26f12239254. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.401295   803 tablet_service.cc:1468] Processing CreateTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 (DEFAULT_TABLE table=default.Table_229010e2d61b49ee9e17f2d21ad323c6 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.401839   965 tablet_service.cc:1468] Processing CreateTablet for tablet b8b1e09d0f224c10ab8df26f12239254 (DEFAULT_TABLE table=default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.402302   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef449bd95fc64bf28c854b03a219c7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.402874   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b8b1e09d0f224c10ab8df26f12239254. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.409004   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8d2b2eef04ee43419aa614b4174b338e" } new_table_name: "default.Table_449a4dab376042d482f1721b6e4c810f" modify_external_catalogs: true
I20250628 07:52:11.412027   964 tablet_service.cc:1468] Processing CreateTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 (DEFAULT_TABLE table=default.Table_229010e2d61b49ee9e17f2d21ad323c6 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.413198   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef449bd95fc64bf28c854b03a219c7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.414011   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d2c7ee9d1fa644da9621c878e540a265"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.415477   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d2c7ee9d1fa644da9621c878e540a265 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.417876  1108 tablet_service.cc:1468] Processing CreateTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 (DEFAULT_TABLE table=default.Table_229010e2d61b49ee9e17f2d21ad323c6 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.419021  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef449bd95fc64bf28c854b03a219c7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.419512  1246 ts_tablet_manager.cc:1905] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.470868   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2770a34952aa436fb1566f2efc41b514" } modify_external_catalogs: true
I20250628 07:52:11.471410   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2770a34952aa436fb1566f2efc41b514" } modify_external_catalogs: true
I20250628 07:52:11.472157   803 tablet_service.cc:1468] Processing CreateTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c (DEFAULT_TABLE table=default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.473372   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f7e2b290c69467aa42ca6328307bc3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.478296   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d37b73f207d7448788b92ebc136a5784"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.479918   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d37b73f207d7448788b92ebc136a5784 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.481545   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f45d9c2ec2244a54b6179c3ea03dd1c0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.482765   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f45d9c2ec2244a54b6179c3ea03dd1c0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.487126   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_900934bdf61248cca0ef6d9dc5e08359" } new_table_name: "default.Table_55bc81989e9b4aa893fee4aa78413057" modify_external_catalogs: true
I20250628 07:52:11.497275  1234 raft_consensus.cc:357] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.498191  1234 raft_consensus.cc:383] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.498471  1234 raft_consensus.cc:738] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.500716   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fbd97bd5efec4a989374c6c08e312e08
I20250628 07:52:11.507297   803 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:11.507897   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 63348a64bb7044dbad9065afff179fbc consensus state has no leader
W20250628 07:52:11.509966   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fbd97bd5efec4a989374c6c08e312e08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fbd97bd5efec4a989374c6c08e312e08 already in progress: creating tablet
I20250628 07:52:11.518556   963 tablet_service.cc:1468] Processing CreateTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c (DEFAULT_TABLE table=default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.520395   962 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:11.521332   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f7e2b290c69467aa42ca6328307bc3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.532546  1234 consensus_queue.cc:260] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.534382  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.536150  1229 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:11.521432  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c (DEFAULT_TABLE table=default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.540066   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6385d021e7bb43a6912ee49dd547295b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.541824   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6385d021e7bb43a6912ee49dd547295b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.542747  1229 tablet_bootstrap.cc:654] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.547698  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f7e2b290c69467aa42ca6328307bc3c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.564278   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_449a4dab376042d482f1721b6e4c810f" } modify_external_catalogs: true
I20250628 07:52:11.564817   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_449a4dab376042d482f1721b6e4c810f" } modify_external_catalogs: true
I20250628 07:52:11.575516  1230 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:11.576002  1230 ts_tablet_manager.cc:1397] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.228s	user 0.012s	sys 0.012s
I20250628 07:52:11.577136  1245 ts_tablet_manager.cc:1918] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:11.579007  1230 raft_consensus.cc:357] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.579735  1230 raft_consensus.cc:383] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.580041  1230 raft_consensus.cc:738] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.580798  1230 consensus_queue.cc:260] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.577559  1245 log.cc:1199] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:11.586221  1245 ts_tablet_manager.cc:1939] T 80f812df4f15406ebe0311c19bb654a7 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:11.592595   986 raft_consensus.cc:1273] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Refusing update from remote peer 6a58514e01ea4989809bc2992fb2b806: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:11.593729   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 80f812df4f15406ebe0311c19bb654a7 (table default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]) successfully deleted
W20250628 07:52:11.594426   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fbd97bd5efec4a989374c6c08e312e08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fbd97bd5efec4a989374c6c08e312e08 already in progress: creating tablet
W20250628 07:52:11.595805  1062 consensus_peers.cc:489] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:11.597610  1255 consensus_queue.cc:1035] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:11.612816  1229 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:11.614508  1229 ts_tablet_manager.cc:1397] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.079s	user 0.005s	sys 0.014s
W20250628 07:52:11.621989   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 63348a64bb7044dbad9065afff179fbc consensus state has no leader
I20250628 07:52:11.623735   963 tablet_service.cc:1468] Processing CreateTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (DEFAULT_TABLE table=default.Table_d2c7ee9d1fa644da9621c878e540a265 [id=853998210b39471f9d2d0bf0ebf072d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.624012   803 tablet_service.cc:1468] Processing CreateTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (DEFAULT_TABLE table=default.Table_d2c7ee9d1fa644da9621c878e540a265 [id=853998210b39471f9d2d0bf0ebf072d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.624631  1246 ts_tablet_manager.cc:1918] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:11.624895   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3637ab35a2b74ce3a111c77cbec8b0b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.624986   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3637ab35a2b74ce3a111c77cbec8b0b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.614619  1230 ts_tablet_manager.cc:1428] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.038s	user 0.003s	sys 0.003s
I20250628 07:52:11.628995  1230 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:11.635591   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 93d993bfdab94f9b8803a5901b1df769
I20250628 07:52:11.639008  1229 raft_consensus.cc:357] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.643730  1234 ts_tablet_manager.cc:1428] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.297s	user 0.022s	sys 0.005s
I20250628 07:52:11.644229   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 93d993bfdab94f9b8803a5901b1df769 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:11.644767  1234 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:11.645022  1244 tablet_replica.cc:331] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:11.645359   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6e7d61663821420fb5bf9aa11076b8b1" } new_table_name: "default.Table_8c6a7fbba6c24b8181422482c95fe4b7" modify_external_catalogs: true
I20250628 07:52:11.645748  1244 raft_consensus.cc:2241] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.646270  1244 raft_consensus.cc:2270] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:11.650239  1244 ts_tablet_manager.cc:1905] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.625037  1246 log.cc:1199] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1cdae9cac7364880a715726890f23485
I20250628 07:52:11.642858   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 93d993bfdab94f9b8803a5901b1df769 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:11.652802  1245 tablet_replica.cc:331] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:11.652940  1246 ts_tablet_manager.cc:1939] T 1cdae9cac7364880a715726890f23485 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:11.653734  1229 raft_consensus.cc:383] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.654083  1229 raft_consensus.cc:738] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.655059   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_181d43b0aeef45e2bc96b51b8392b136"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:11.656431   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1cdae9cac7364880a715726890f23485 (table default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]) successfully deleted
W20250628 07:52:11.656868   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_181d43b0aeef45e2bc96b51b8392b136 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.657454  1245 raft_consensus.cc:2241] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:11.657704  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 93d993bfdab94f9b8803a5901b1df769 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:11.656038  1229 consensus_queue.cc:260] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.663549  1230 tablet_bootstrap.cc:654] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.670511  1245 pending_rounds.cc:70] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 2 pending ops.
I20250628 07:52:11.670948  1245 pending_rounds.cc:77] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 2 } timestamp: 7172493850948472832 op_type: ALTER_SCHEMA_OP alter_schema_request { tablet_id: "93d993bfdab94f9b8803a5901b1df769" schema { columns { id: 10 name: "key" type: INT32 is_key: true is_nullable: false encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } schema_version: 3 new_table_name: "default.Table_8d2b2eef04ee43419aa614b4174b338e" dest_uuid: "6a58514e01ea4989809bc2992fb2b806" new_extra_config { } }
I20250628 07:52:11.672768  1245 pending_rounds.cc:77] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493849726078976 op_type: NO_OP noop_request { }
I20250628 07:52:11.673437  1245 raft_consensus.cc:2887] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:11.673966  1245 raft_consensus.cc:2270] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:11.676198  1229 ts_tablet_manager.cc:1428] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.061s	user 0.004s	sys 0.004s
W20250628 07:52:11.683522  1062 consensus_peers.cc:489] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:11.680008  1245 ts_tablet_manager.cc:1905] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.690155  1246 tablet_replica.cc:331] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:11.693243  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (DEFAULT_TABLE table=default.Table_d2c7ee9d1fa644da9621c878e540a265 [id=853998210b39471f9d2d0bf0ebf072d5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.694327  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3637ab35a2b74ce3a111c77cbec8b0b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.695772  1229 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:11.676227   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fbd97bd5efec4a989374c6c08e312e08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fbd97bd5efec4a989374c6c08e312e08 already in progress: creating tablet
I20250628 07:52:11.708530   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6385d021e7bb43a6912ee49dd547295b" } modify_external_catalogs: true
I20250628 07:52:11.709069   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6385d021e7bb43a6912ee49dd547295b" } modify_external_catalogs: true
I20250628 07:52:11.714238   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_f45d9c2ec2244a54b6179c3ea03dd1c0" } new_table_name: "default.Table_ed752c51c98743c6b11f0f3a2ce3db69" modify_external_catalogs: true
I20250628 07:52:11.719583   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_801ed4c53ecc49cb9a54dd6547a8da6d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.721674   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_801ed4c53ecc49cb9a54dd6547a8da6d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.733781  1246 raft_consensus.cc:2241] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:11.737854  1246 raft_consensus.cc:2270] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:11.738554   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 6e912d18dcff4a938a867ea06ee7e0f0 consensus state has no leader
I20250628 07:52:11.738950  1266 tablet.cc:1722] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:11.742229   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_712625afc1034446a1f0bfd3618ea308"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.743702   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_712625afc1034446a1f0bfd3618ea308 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.743942   541 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet e7d36bb5738945c98b97bd27d04186ce
I20250628 07:52:11.751050  1230 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:11.751531  1230 ts_tablet_manager.cc:1397] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.123s	user 0.009s	sys 0.004s
I20250628 07:52:11.754371  1230 raft_consensus.cc:357] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.762362  1230 raft_consensus.cc:383] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.762701  1230 raft_consensus.cc:738] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.763433  1230 consensus_queue.cc:260] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.765708  1230 ts_tablet_manager.cc:1428] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.014s	user 0.005s	sys 0.001s
I20250628 07:52:11.774319  1230 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:11.778354  1259 tablet_service.cc:696] failed op from {username='slave'} at 127.0.0.1:54948: Illegal state: RaftConsensus is not running: State = Stopped
W20250628 07:52:11.784405  1259 tablet_service.cc:696] failed op from {username='slave'} at 127.0.0.1:54948: Illegal state: RaftConsensus is not running: State = Stopped
I20250628 07:52:11.787669  1230 tablet_bootstrap.cc:654] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.794800  1246 ts_tablet_manager.cc:1905] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:11.804495  1234 tablet_bootstrap.cc:654] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.814376  1229 tablet_bootstrap.cc:654] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.820417  1246 ts_tablet_manager.cc:1918] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:11.820791  1246 log.cc:1199] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/93d993bfdab94f9b8803a5901b1df769
I20250628 07:52:11.821710  1246 ts_tablet_manager.cc:1939] T 93d993bfdab94f9b8803a5901b1df769 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:11.823467   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]). Expected version 1 got 0
I20250628 07:52:11.825826   545 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
I20250628 07:52:11.829929  1105 tablet_service.cc:1468] Processing CreateTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (DEFAULT_TABLE table=default.Table_d37b73f207d7448788b92ebc136a5784 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.831243  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.834769   545 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]). Expected version 1 got 0
W20250628 07:52:11.834947   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 84a8065c2d1349b792308ac3aec1a8fe consensus state has no leader
I20250628 07:52:11.834334   803 tablet_service.cc:1468] Processing CreateTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (DEFAULT_TABLE table=default.Table_d37b73f207d7448788b92ebc136a5784 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.835944   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.836021   805 tablet_service.cc:1468] Processing CreateTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe (DEFAULT_TABLE table=default.Table_f45d9c2ec2244a54b6179c3ea03dd1c0 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.836941   545 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
I20250628 07:52:11.837337   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84a8065c2d1349b792308ac3aec1a8fe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.847747  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe (DEFAULT_TABLE table=default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.848184   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a147ed0492104e618020dabd5d37a674"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:11.848770  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84a8065c2d1349b792308ac3aec1a8fe. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:11.849735   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a147ed0492104e618020dabd5d37a674 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.851569   966 tablet_service.cc:1468] Processing CreateTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe (DEFAULT_TABLE table=default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.852607   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 84a8065c2d1349b792308ac3aec1a8fe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.861542   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_448f6b4c3c354e76aaf65492579adba1" } modify_external_catalogs: true
I20250628 07:52:11.862027   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_448f6b4c3c354e76aaf65492579adba1" } modify_external_catalogs: true
I20250628 07:52:11.874778  1230 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:11.875267  1230 ts_tablet_manager.cc:1397] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.101s	user 0.014s	sys 0.026s
I20250628 07:52:11.878273  1230 raft_consensus.cc:357] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.878993  1230 raft_consensus.cc:383] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:11.879302  1230 raft_consensus.cc:738] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:11.879146   963 tablet_service.cc:1468] Processing CreateTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (DEFAULT_TABLE table=default.Table_d37b73f207d7448788b92ebc136a5784 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:11.880122   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:11.880000  1230 consensus_queue.cc:260] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.885912  1230 ts_tablet_manager.cc:1428] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.000s
I20250628 07:52:11.889933  1230 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:11.890487  1237 raft_consensus.cc:491] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:11.890967  1237 raft_consensus.cc:513] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.892664  1237 leader_election.cc:290] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:11.894078  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "752fc37e7210458fb0c631d01204a3bb" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:11.894646  1129 raft_consensus.cc:2466] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:11.895376   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_449a4dab376042d482f1721b6e4c810f [id=4a76c0057963463f84ebb91d85f9725e]) successfully deleted
I20250628 07:52:11.896457   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "752fc37e7210458fb0c631d01204a3bb" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:11.897787   921 leader_election.cc:304] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:11.898504  1255 raft_consensus.cc:491] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:11.899483  1255 raft_consensus.cc:513] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.901325  1255 leader_election.cc:290] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:11.902343   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:11.906888   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:11.907482   986 raft_consensus.cc:2466] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
W20250628 07:52:11.908346   920 leader_election.cc:343] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.909981  1237 raft_consensus.cc:2802] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:11.910508  1237 raft_consensus.cc:491] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:11.913079  1237 raft_consensus.cc:3058] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:11.914907  1062 leader_election.cc:343] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.915145   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ff73d822bbd947e38b75f3f0cfc65c89"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:11.915628  1062 leader_election.cc:304] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:11.916726   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ff73d822bbd947e38b75f3f0cfc65c89 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.917266  1230 tablet_bootstrap.cc:654] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:11.917726  1255 raft_consensus.cc:2802] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:11.923197  1255 raft_consensus.cc:491] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:11.923552  1255 raft_consensus.cc:3058] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:11.930627  1255 raft_consensus.cc:513] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.935153  1255 leader_election.cc:290] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:11.937565   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:11.938118   986 raft_consensus.cc:3058] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:11.940115  1237 raft_consensus.cc:513] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.942384   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a147ed0492104e618020dabd5d37a674" } new_table_name: "default.Table_18d9b7c303dd43c68262a4c621f81364" modify_external_catalogs: true
I20250628 07:52:11.950150   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "752fc37e7210458fb0c631d01204a3bb" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:11.952324  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "752fc37e7210458fb0c631d01204a3bb" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:11.953437   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:11.952880  1129 raft_consensus.cc:3058] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:11.962378   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_181d43b0aeef45e2bc96b51b8392b136" } modify_external_catalogs: true
I20250628 07:52:11.962929   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_181d43b0aeef45e2bc96b51b8392b136" } modify_external_catalogs: true
W20250628 07:52:11.965164  1062 leader_election.cc:343] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.950793  1237 leader_election.cc:290] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:11.953423   920 leader_election.cc:343] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:11.948791  1244 ts_tablet_manager.cc:1918] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:11.967806  1244 log.cc:1199] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/93d993bfdab94f9b8803a5901b1df769
I20250628 07:52:11.968668  1244 ts_tablet_manager.cc:1939] T 93d993bfdab94f9b8803a5901b1df769 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:11.974771  1129 raft_consensus.cc:2466] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:11.975911   921 leader_election.cc:304] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:11.981329  1237 raft_consensus.cc:2802] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:11.983235   986 raft_consensus.cc:2466] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:11.984400  1234 tablet_bootstrap.cc:492] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:11.984329  1062 leader_election.cc:304] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:11.985141  1255 raft_consensus.cc:2802] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:11.985558  1255 raft_consensus.cc:695] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:11.986284  1255 consensus_queue.cc:237] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:11.986490   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_344e153bf04b4bf19e388f9f092ce6fb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:11.988150   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_344e153bf04b4bf19e388f9f092ce6fb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:11.984851  1234 ts_tablet_manager.cc:1397] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.340s	user 0.008s	sys 0.005s
I20250628 07:52:11.997246  1234 raft_consensus.cc:357] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.003208  1237 raft_consensus.cc:695] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:12.005076   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_449a4dab376042d482f1721b6e4c810f [id=4a76c0057963463f84ebb91d85f9725e]) successfully deleted
I20250628 07:52:12.004778  1237 consensus_queue.cc:237] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.006986  1245 ts_tablet_manager.cc:1918] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:12.007995  1234 raft_consensus.cc:383] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.014468  1104 tablet_service.cc:1468] Processing CreateTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (DEFAULT_TABLE table=default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.015548  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.015663   545 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
I20250628 07:52:12.017699  1234 raft_consensus.cc:738] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.017797   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:12.021992   803 tablet_service.cc:1468] Processing CreateTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (DEFAULT_TABLE table=default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.022329   805 tablet_service.cc:1468] Processing CreateTablet for tablet 0ac2a2d045864749841a9b5de03d140c (DEFAULT_TABLE table=default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.023028   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.025835  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 0ac2a2d045864749841a9b5de03d140c (DEFAULT_TABLE table=default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.026823  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0ac2a2d045864749841a9b5de03d140c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.032434  1230 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.032835  1230 ts_tablet_manager.cc:1397] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.143s	user 0.015s	sys 0.035s
I20250628 07:52:12.035894  1230 raft_consensus.cc:357] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.037571  1230 raft_consensus.cc:383] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.037925  1230 raft_consensus.cc:738] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.038156   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0ac2a2d045864749841a9b5de03d140c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.038686  1230 consensus_queue.cc:260] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.042006   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d37b73f207d7448788b92ebc136a5784" } new_table_name: "default.Table_9863b171d29f48d88a2a547726aa8543" modify_external_catalogs: true
I20250628 07:52:12.042337  1230 ts_tablet_manager.cc:1428] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.009s	user 0.005s	sys 0.002s
I20250628 07:52:12.045746   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:12.046540  1230 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.046448   547 catalog_manager.cc:5582] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } }
I20250628 07:52:12.047679   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
W20250628 07:52:12.058540   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2d77be1f37d2400f842c01019e4eb474 has no consensus state
I20250628 07:52:12.065140   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:12.067165   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.067966  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.068174  1230 tablet_bootstrap.cc:654] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:12.070794   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0ac2a2d045864749841a9b5de03d140c because the tablet was not found. No further retry: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
W20250628 07:52:12.074724   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0ac2a2d045864749841a9b5de03d140c because the tablet was not found. No further retry: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:12.023955   963 tablet_service.cc:1468] Processing CreateTablet for tablet 0ac2a2d045864749841a9b5de03d140c (DEFAULT_TABLE table=default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.081727  1245 log.cc:1199] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/93d993bfdab94f9b8803a5901b1df769
I20250628 07:52:12.082302   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0ac2a2d045864749841a9b5de03d140c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.082499  1245 ts_tablet_manager.cc:1939] T 93d993bfdab94f9b8803a5901b1df769 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:12.082201   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c6657dd54c2f4decb3eda54e473d4c10"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.083820   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c6657dd54c2f4decb3eda54e473d4c10 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.020560  1234 consensus_queue.cc:260] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.087898   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 93d993bfdab94f9b8803a5901b1df769 (table default.Table_449a4dab376042d482f1721b6e4c810f [id=4a76c0057963463f84ebb91d85f9725e]) successfully deleted
I20250628 07:52:12.093374  1234 ts_tablet_manager.cc:1428] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.102s	user 0.002s	sys 0.004s
I20250628 07:52:12.029845   966 tablet_service.cc:1468] Processing CreateTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (DEFAULT_TABLE table=default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.068611   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.097962   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.099464   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet b8b1e09d0f224c10ab8df26f12239254
W20250628 07:52:12.102370   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 consensus state has no leader
I20250628 07:52:12.104552   804 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.105223   965 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.105883  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:12.108421   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b8b1e09d0f224c10ab8df26f12239254 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b8b1e09d0f224c10ab8df26f12239254 already in progress: creating tablet
I20250628 07:52:12.108914  1230 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.110376  1230 ts_tablet_manager.cc:1397] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.064s	user 0.007s	sys 0.007s
I20250628 07:52:12.113849  1230 raft_consensus.cc:357] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.114543  1230 raft_consensus.cc:383] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.114847  1230 raft_consensus.cc:738] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:12.115680   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b8b1e09d0f224c10ab8df26f12239254 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b8b1e09d0f224c10ab8df26f12239254 already in progress: creating tablet
I20250628 07:52:12.115403  1230 consensus_queue.cc:260] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:12.120162   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0ac2a2d045864749841a9b5de03d140c because the tablet was not found. No further retry: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
W20250628 07:52:12.121616   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet b8b1e09d0f224c10ab8df26f12239254 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b8b1e09d0f224c10ab8df26f12239254 already in progress: creating tablet
I20250628 07:52:12.127198  1230 ts_tablet_manager.cc:1428] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.000s
I20250628 07:52:12.133419  1230 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.138648  1229 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:12.139052  1229 ts_tablet_manager.cc:1397] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.444s	user 0.015s	sys 0.001s
I20250628 07:52:12.141168  1229 raft_consensus.cc:357] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.146288  1229 raft_consensus.cc:383] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.146605  1229 raft_consensus.cc:738] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.151206   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111), 
I20250628 07:52:12.151880   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:12.151685  1229 consensus_queue.cc:260] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.155472  1230 tablet_bootstrap.cc:654] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.161695  1229 ts_tablet_manager.cc:1428] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.022s	user 0.007s	sys 0.001s
I20250628 07:52:12.169756  1234 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:12.174286  1229 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:12.175206   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_682dd2d7af414997aa391924c47d7b31"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.176916   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_682dd2d7af414997aa391924c47d7b31 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.184271   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cdae9cac7364880a715726890f23485 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.185099  1244 tablet_replica.cc:331] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:12.185717  1244 raft_consensus.cc:2241] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.186128  1244 raft_consensus.cc:2270] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.196976  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.197875  1246 tablet_replica.cc:331] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:12.198616  1246 raft_consensus.cc:2241] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.199074  1246 raft_consensus.cc:2270] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.201346  1246 ts_tablet_manager.cc:1905] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.204537  1229 tablet_bootstrap.cc:654] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.206411  1230 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.206831  1230 ts_tablet_manager.cc:1397] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.078s	user 0.010s	sys 0.000s
I20250628 07:52:12.209014  1230 raft_consensus.cc:357] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.209698  1230 raft_consensus.cc:383] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.209939  1230 raft_consensus.cc:738] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.210479  1230 consensus_queue.cc:260] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.212321  1230 ts_tablet_manager.cc:1428] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:12.213078  1230 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.217151   986 raft_consensus.cc:1273] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Refusing update from remote peer 6a58514e01ea4989809bc2992fb2b806: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:12.221139  1235 consensus_queue.cc:1035] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:12.230695  1230 tablet_bootstrap.cc:654] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.241950  1230 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.242327  1230 ts_tablet_manager.cc:1397] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.029s	user 0.010s	sys 0.000s
I20250628 07:52:12.246779  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.247565  1230 raft_consensus.cc:357] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.249068  1230 raft_consensus.cc:383] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.249342  1230 raft_consensus.cc:738] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.249900  1230 consensus_queue.cc:260] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.253250   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a70f9c604ca44b7eae2b205bc80fdf1d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.254967   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a70f9c604ca44b7eae2b205bc80fdf1d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.257602   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_de1b1ac5b4e44da284d4cf266fef4876"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:12.258175  1230 ts_tablet_manager.cc:1428] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.016s	user 0.003s	sys 0.003s
W20250628 07:52:12.259428   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_de1b1ac5b4e44da284d4cf266fef4876 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.263746   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ff73d822bbd947e38b75f3f0cfc65c89" } new_table_name: "default.Table_1c78cba777ca43b6b253acb1d9c35a3a" modify_external_catalogs: true
I20250628 07:52:12.271523  1230 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.272644   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_801ed4c53ecc49cb9a54dd6547a8da6d" } modify_external_catalogs: true
I20250628 07:52:12.273247   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_801ed4c53ecc49cb9a54dd6547a8da6d" } modify_external_catalogs: true
I20250628 07:52:12.278908  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet b5f50ad0815b433287d36bb209bee50d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.283650  1230 tablet_bootstrap.cc:654] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.304965   966 tablet_service.cc:1468] Processing CreateTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 (DEFAULT_TABLE table=default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.306070   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet deb5a9ae7ab5463cae35ff1086fd8348. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.306778  1105 tablet_service.cc:1468] Processing CreateTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 (DEFAULT_TABLE table=default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.308051  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet deb5a9ae7ab5463cae35ff1086fd8348. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.310375   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9c39cbe761084240846a9147b9666d33" } modify_external_catalogs: true
I20250628 07:52:12.310931   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9c39cbe761084240846a9147b9666d33" } modify_external_catalogs: true
I20250628 07:52:12.316008   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_62f8e336893a4e0b8e2f21306fa953b8" } new_table_name: "default.Table_fed6fabd13364e748240a44b749e4158" modify_external_catalogs: true
I20250628 07:52:12.322182  1280 raft_consensus.cc:491] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:12.322691   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 0f22700fc70d4f14ae51a834900db22b has no consensus state
I20250628 07:52:12.322672  1280 raft_consensus.cc:513] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.325495   805 tablet_service.cc:1468] Processing CreateTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 (DEFAULT_TABLE table=default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.326610   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet deb5a9ae7ab5463cae35ff1086fd8348. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.339852  1244 ts_tablet_manager.cc:1905] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:12.342804  1062 consensus_peers.cc:489] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:12.347070   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:12.348528   804 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.349012   963 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.349707  1280 leader_election.cc:290] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:12.350544   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:12.351070   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 already in progress: creating tablet
W20250628 07:52:12.351950  1062 leader_election.cc:343] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.352986   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:12.353296  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:12.354386  1062 leader_election.cc:343] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.354786  1062 leader_election.cc:304] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:12.359350  1280 raft_consensus.cc:2747] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:12.387169  1234 tablet_bootstrap.cc:654] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.391706   804 tablet_service.cc:1468] Processing CreateTablet for tablet 2d77be1f37d2400f842c01019e4eb474 (DEFAULT_TABLE table=default.Table_18d9b7c303dd43c68262a4c621f81364 [id=15ceb9913eb84e44b89a2393cc7da9e3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.392848  1280 raft_consensus.cc:491] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:12.393015   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d77be1f37d2400f842c01019e4eb474. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.393262  1280 raft_consensus.cc:513] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.395093  1280 leader_election.cc:290] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
W20250628 07:52:12.397544  1062 leader_election.cc:343] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:12.398818   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "80f812df4f15406ebe0311c19bb654a7" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:12.400063  1062 leader_election.cc:343] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.400439  1062 leader_election.cc:304] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:12.401181  1280 raft_consensus.cc:2747] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:12.422798  1229 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:12.423199  1229 ts_tablet_manager.cc:1397] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.249s	user 0.014s	sys 0.046s
I20250628 07:52:12.423885   966 tablet_service.cc:1468] Processing CreateTablet for tablet 2d77be1f37d2400f842c01019e4eb474 (DEFAULT_TABLE table=default.Table_18d9b7c303dd43c68262a4c621f81364 [id=15ceb9913eb84e44b89a2393cc7da9e3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.425006   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 752fc37e7210458fb0c631d01204a3bb
I20250628 07:52:12.425249   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d77be1f37d2400f842c01019e4eb474. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.425459  1229 raft_consensus.cc:357] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.425997  1229 raft_consensus.cc:383] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.426200  1229 raft_consensus.cc:738] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.427170  1229 consensus_queue.cc:260] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.428165   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 752fc37e7210458fb0c631d01204a3bb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.429001  1229 ts_tablet_manager.cc:1428] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20250628 07:52:12.429359   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 752fc37e7210458fb0c631d01204a3bb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.429440  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet 752fc37e7210458fb0c631d01204a3bb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.429801  1229 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:12.430495  1245 tablet_replica.cc:331] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:12.431294  1245 raft_consensus.cc:2241] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:12.432013  1245 pending_rounds.cc:70] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:12.432207  1245 pending_rounds.cc:77] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493852753166336 op_type: NO_OP noop_request { }
I20250628 07:52:12.432504  1245 raft_consensus.cc:2887] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:12.432834  1245 raft_consensus.cc:2270] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:12.433537  1284 log.cc:927] Time spent T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Append to log took a long time: real 0.100s	user 0.000s	sys 0.000s
I20250628 07:52:12.444859   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:12.445606   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
I20250628 07:52:12.448858  1281 mvcc.cc:204] Tried to move back new op lower bound from 7172493853543538688 to 7172493852625252352. Current Snapshot: MvccSnapshot[applied={T|T < 7172493853543538688}]
I20250628 07:52:12.453117  1229 tablet_bootstrap.cc:654] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.454780  1245 ts_tablet_manager.cc:1905] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.479758   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_55bc81989e9b4aa893fee4aa78413057 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 2 got 0
I20250628 07:52:12.480394   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c2ddeca0b08444d5b48e4c567ace5c35"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.481945   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c2ddeca0b08444d5b48e4c567ace5c35 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.483135  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 2d77be1f37d2400f842c01019e4eb474 (DEFAULT_TABLE table=default.Table_18d9b7c303dd43c68262a4c621f81364 [id=15ceb9913eb84e44b89a2393cc7da9e3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.474387  1285 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:12.484131  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d77be1f37d2400f842c01019e4eb474. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.483871  1230 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.485558  1230 ts_tablet_manager.cc:1397] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.214s	user 0.035s	sys 0.061s
I20250628 07:52:12.484561  1286 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:12.489732  1229 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:12.490136  1229 ts_tablet_manager.cc:1397] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.060s	user 0.010s	sys 0.000s
I20250628 07:52:12.492866  1229 raft_consensus.cc:357] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:12.495622   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 82865a5c5fc04d82a6be4c1d60fe9d7e consensus state has no leader
I20250628 07:52:12.497964  1230 raft_consensus.cc:357] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.501312  1230 raft_consensus.cc:383] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.502545  1230 raft_consensus.cc:738] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.505559   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_26605bb2e3db42a78e439ffdd1f85944"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.507480   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_26605bb2e3db42a78e439ffdd1f85944 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.506728  1230 consensus_queue.cc:260] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.516235   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9753204a41484dce8870a3bdc8fdc6fb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.517858   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9753204a41484dce8870a3bdc8fdc6fb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.519659  1230 ts_tablet_manager.cc:1428] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.033s	user 0.007s	sys 0.001s
I20250628 07:52:12.523815  1230 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.525492  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.529206   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_55bc81989e9b4aa893fee4aa78413057" } new_table_name: "default.Table_78d794b2cff54bdf88d1996d140e4de8" modify_external_catalogs: true
I20250628 07:52:12.532231  1259 mvcc.cc:204] Tried to move back new op lower bound from 7172493853543538688 to 7172493852625252352. Current Snapshot: MvccSnapshot[applied={T|T < 7172493853543538688}]
I20250628 07:52:12.541199  1230 tablet_bootstrap.cc:654] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.543771  1285 tablet.cc:1717] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Skipping requested alter to schema version 2, tablet already version 2
I20250628 07:52:12.520641  1245 ts_tablet_manager.cc:1918] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:12.552270  1245 log.cc:1199] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/752fc37e7210458fb0c631d01204a3bb
I20250628 07:52:12.552462   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_fed6fabd13364e748240a44b749e4158" } modify_external_catalogs: true
I20250628 07:52:12.553020  1245 ts_tablet_manager.cc:1939] T 752fc37e7210458fb0c631d01204a3bb P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:12.552999   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_fed6fabd13364e748240a44b749e4158" } modify_external_catalogs: true
I20250628 07:52:12.553597  1229 raft_consensus.cc:383] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.561534  1229 raft_consensus.cc:738] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.562283  1229 consensus_queue.cc:260] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.564340  1229 ts_tablet_manager.cc:1428] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.074s	user 0.006s	sys 0.000s
I20250628 07:52:12.569341  1285 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:12.577337  1246 ts_tablet_manager.cc:1918] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:12.577751  1246 log.cc:1199] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fbd97bd5efec4a989374c6c08e312e08
I20250628 07:52:12.578399  1246 ts_tablet_manager.cc:1939] T fbd97bd5efec4a989374c6c08e312e08 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:12.580323   963 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.581143   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 3 got 2
I20250628 07:52:12.585547  1244 ts_tablet_manager.cc:1918] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:12.586053  1244 log.cc:1199] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/1cdae9cac7364880a715726890f23485
I20250628 07:52:12.586773  1244 ts_tablet_manager.cc:1939] T 1cdae9cac7364880a715726890f23485 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:12.599507  1245 tablet_replica.cc:331] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:12.600075  1229 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:12.600247  1245 raft_consensus.cc:2241] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.600732  1245 raft_consensus.cc:2270] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.601465   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]) successfully deleted
I20250628 07:52:12.604004   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 3 got 2
I20250628 07:52:12.604306  1245 ts_tablet_manager.cc:1905] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.605672  1229 tablet_bootstrap.cc:654] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.611616  1234 tablet_bootstrap.cc:492] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:12.612074  1234 ts_tablet_manager.cc:1397] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.443s	user 0.002s	sys 0.009s
I20250628 07:52:12.614423   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 82865a5c5fc04d82a6be4c1d60fe9d7e
I20250628 07:52:12.617388  1230 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.618108  1230 ts_tablet_manager.cc:1397] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.095s	user 0.017s	sys 0.002s
I20250628 07:52:12.619685   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.622767  1234 raft_consensus.cc:357] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.623437  1234 raft_consensus.cc:383] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.624274  1234 raft_consensus.cc:738] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.625726  1280 raft_consensus.cc:491] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:12.625761  1235 raft_consensus.cc:491] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:12.626533  1280 raft_consensus.cc:513] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.626569  1235 raft_consensus.cc:513] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.628407  1235 leader_election.cc:290] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:12.628502  1280 leader_election.cc:290] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:12.628361  1234 consensus_queue.cc:260] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.630322   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c1a06bbe216f419598a587056f9b8d05" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:12.632485  1062 leader_election.cc:343] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.631577  1230 raft_consensus.cc:357] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.633044  1230 raft_consensus.cc:383] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.633567  1230 raft_consensus.cc:738] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.634126   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b5f50ad0815b433287d36bb209bee50d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:12.634114   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_682dd2d7af414997aa391924c47d7b31" } modify_external_catalogs: true
I20250628 07:52:12.634642   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_682dd2d7af414997aa391924c47d7b31" } modify_external_catalogs: true
I20250628 07:52:12.634723  1230 consensus_queue.cc:260] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
W20250628 07:52:12.635633  1062 leader_election.cc:343] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.639353   544 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f] alter complete (version 3)
W20250628 07:52:12.655098   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (table default.Table_fed6fabd13364e748240a44b749e4158 [id=0678e3fca14f4accbcf09e6ff76db055], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 82865a5c5fc04d82a6be4c1d60fe9d7e consensus state has no leader
I20250628 07:52:12.656790  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.659125  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.661170  1234 ts_tablet_manager.cc:1428] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.049s	user 0.007s	sys 0.008s
I20250628 07:52:12.667733  1237 raft_consensus.cc:491] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:12.668174  1237 raft_consensus.cc:513] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.670325   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 1cdae9cac7364880a715726890f23485 (table default.Table_3bd28637c21b4d4881c1880f63a56b85 [id=2487c6eaaca64c1aae6b2658f4e59054]) successfully deleted
I20250628 07:52:12.680586   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fbd97bd5efec4a989374c6c08e312e08 (table default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]) successfully deleted
I20250628 07:52:12.688190  1246 tablet_replica.cc:331] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:12.689842   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:12.690564  1246 raft_consensus.cc:2241] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.691208  1234 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:12.691114   920 leader_election.cc:343] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.691442  1246 raft_consensus.cc:2270] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.692773  1237 leader_election.cc:290] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:12.692992  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:12.693655  1129 raft_consensus.cc:2466] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:12.694957   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.694829   921 leader_election.cc:304] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:12.697306   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 already in progress: creating tablet
I20250628 07:52:12.698890  1286 tablet.cc:1717] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Skipping requested alter to schema version 2, tablet already version 2
I20250628 07:52:12.699726   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_85234bd18f8e42a3b62dc723411c065a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:12.700091  1237 raft_consensus.cc:2802] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:12.699971   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_229010e2d61b49ee9e17f2d21ad323c6" } new_table_name: "default.Table_5b9c6a7df6f04b808eecd7ee86d425b4" modify_external_catalogs: true
I20250628 07:52:12.700402  1237 raft_consensus.cc:491] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:12.700731  1237 raft_consensus.cc:3058] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:12.701447   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_85234bd18f8e42a3b62dc723411c065a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.701907  1244 tablet_replica.cc:331] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:12.702632  1244 raft_consensus.cc:2241] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.703152  1244 raft_consensus.cc:2270] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.705432   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c1a06bbe216f419598a587056f9b8d05" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:12.707839  1062 leader_election.cc:343] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.708230  1062 leader_election.cc:304] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:12.709225  1280 raft_consensus.cc:2747] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:12.710166  1062 leader_election.cc:343] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: b5f50ad0815b433287d36bb209bee50d
I20250628 07:52:12.710542  1062 leader_election.cc:304] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:12.711475  1280 raft_consensus.cc:2747] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:12.718470   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:12.720280  1245 ts_tablet_manager.cc:1918] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:12.720806  1245 log.cc:1199] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fbd97bd5efec4a989374c6c08e312e08
I20250628 07:52:12.721479  1229 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:12.721871  1229 ts_tablet_manager.cc:1397] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.122s	user 0.010s	sys 0.000s
I20250628 07:52:12.724071  1229 raft_consensus.cc:357] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.724682  1229 raft_consensus.cc:383] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.725914  1230 ts_tablet_manager.cc:1428] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.107s	user 0.007s	sys 0.000s
W20250628 07:52:12.727214   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet ef449bd95fc64bf28c854b03a219c7b4 consensus state has no leader
I20250628 07:52:12.727538  1245 ts_tablet_manager.cc:1939] T fbd97bd5efec4a989374c6c08e312e08 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:12.726889  1286 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:12.724963  1229 raft_consensus.cc:738] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.735586  1234 tablet_bootstrap.cc:654] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.738724  1237 raft_consensus.cc:513] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.741878   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:12.743139   920 leader_election.cc:343] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.743965  1229 consensus_queue.cc:260] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.746367  1234 tablet_bootstrap.cc:492] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:12.746809  1234 ts_tablet_manager.cc:1397] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.056s	user 0.000s	sys 0.010s
I20250628 07:52:12.747700  1244 ts_tablet_manager.cc:1905] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.744488  1230 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.750890  1245 tablet_replica.cc:331] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:12.751680  1245 raft_consensus.cc:2241] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.752187  1245 raft_consensus.cc:2270] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.753006  1246 ts_tablet_manager.cc:1905] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.749886  1234 raft_consensus.cc:357] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.757228  1234 raft_consensus.cc:383] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.757514  1234 raft_consensus.cc:738] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.758133  1234 consensus_queue.cc:260] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.760262  1280 raft_consensus.cc:491] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:12.760690  1280 raft_consensus.cc:513] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.762595  1234 ts_tablet_manager.cc:1428] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.015s	user 0.004s	sys 0.003s
I20250628 07:52:12.762638  1280 leader_election.cc:290] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:12.765113   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "6e912d18dcff4a938a867ea06ee7e0f0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:12.766749  1234 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:12.766852  1062 leader_election.cc:343] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.768222   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "6e912d18dcff4a938a867ea06ee7e0f0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:12.768867   985 raft_consensus.cc:2466] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:12.769912  1062 leader_election.cc:304] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:12.771291  1280 raft_consensus.cc:2802] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:12.771785  1280 raft_consensus.cc:491] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:12.772229  1280 raft_consensus.cc:3058] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:12.777269   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:12.779103   533 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.CreateTablet -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 60348.3us
I20250628 07:52:12.784977  1230 tablet_bootstrap.cc:654] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.786501  1280 raft_consensus.cc:513] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.788811   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "6e912d18dcff4a938a867ea06ee7e0f0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:12.789978  1062 leader_election.cc:343] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:12.790764   804 tablet_service.cc:1515] Processing DeleteTablet for tablet b5f50ad0815b433287d36bb209bee50d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.792205   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c2ddeca0b08444d5b48e4c567ace5c35" } new_table_name: "default.Table_5455ab227ebe42608fed93cfb1b31b5a" modify_external_catalogs: true
I20250628 07:52:12.793066   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_929c4c35bfbb404cafec33d8710f7a1a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.794524   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_929c4c35bfbb404cafec33d8710f7a1a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.795064  1234 tablet_bootstrap.cc:654] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.796077   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:12.797398   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:12.797866  1230 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:12.798230  1230 ts_tablet_manager.cc:1397] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.054s	user 0.009s	sys 0.000s
I20250628 07:52:12.800163   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fbd97bd5efec4a989374c6c08e312e08 (table default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]) successfully deleted
I20250628 07:52:12.800238  1230 raft_consensus.cc:357] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.800777  1230 raft_consensus.cc:383] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.800999  1230 raft_consensus.cc:738] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.801529  1230 consensus_queue.cc:260] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:12.802584  1280 leader_election.cc:290] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:12.803297  1230 ts_tablet_manager.cc:1428] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:12.805776  1230 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:12.812800   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "6e912d18dcff4a938a867ea06ee7e0f0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:12.813400   985 raft_consensus.cc:3058] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:12.813530  1230 tablet_bootstrap.cc:654] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.824343   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_5b9c6a7df6f04b808eecd7ee86d425b4 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 1 got 0
W20250628 07:52:12.829028   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2b51d6076e344c8f8e388b77b4b23928 has no consensus state
W20250628 07:52:12.834185   878 tablet.cc:2378] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:12.834367   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_5b9c6a7df6f04b808eecd7ee86d425b4 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 1 got 0
I20250628 07:52:12.840274  1244 ts_tablet_manager.cc:1918] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:12.840677  1244 log.cc:1199] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/752fc37e7210458fb0c631d01204a3bb
I20250628 07:52:12.840699   804 tablet_service.cc:1468] Processing CreateTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 (DEFAULT_TABLE table=default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.841893   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67a81eb4342740e5bc16ee7f5eacb489. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.843250  1244 ts_tablet_manager.cc:1939] T 752fc37e7210458fb0c631d01204a3bb P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:12.843885  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 (DEFAULT_TABLE table=default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.844538  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 0f22700fc70d4f14ae51a834900db22b (DEFAULT_TABLE table=default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.844838  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67a81eb4342740e5bc16ee7f5eacb489. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.845546  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f22700fc70d4f14ae51a834900db22b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.845961  1244 tablet_replica.cc:331] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:12.846551  1244 raft_consensus.cc:2241] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.847002  1244 raft_consensus.cc:2270] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.847290   805 tablet_service.cc:1468] Processing CreateTablet for tablet 0f22700fc70d4f14ae51a834900db22b (DEFAULT_TABLE table=default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.848160   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f22700fc70d4f14ae51a834900db22b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.853354   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]) successfully deleted
I20250628 07:52:12.853791  1234 tablet_bootstrap.cc:492] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:12.854024   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 82865a5c5fc04d82a6be4c1d60fe9d7e already in progress: creating tablet
I20250628 07:52:12.854173  1234 ts_tablet_manager.cc:1397] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.088s	user 0.006s	sys 0.009s
I20250628 07:52:12.856406  1234 raft_consensus.cc:357] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.857033  1234 raft_consensus.cc:383] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:12.858016  1234 raft_consensus.cc:738] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:12.862538   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9753204a41484dce8870a3bdc8fdc6fb" } modify_external_catalogs: true
I20250628 07:52:12.863095   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9753204a41484dce8870a3bdc8fdc6fb" } modify_external_catalogs: true
I20250628 07:52:12.858777  1234 consensus_queue.cc:260] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.866235  1244 ts_tablet_manager.cc:1905] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:12.867238   803 tablet_service.cc:1468] Processing CreateTablet for tablet 474cec4459954e10a18de031946a2346 (DEFAULT_TABLE table=default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.868247   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474cec4459954e10a18de031946a2346. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.869819  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 474cec4459954e10a18de031946a2346 (DEFAULT_TABLE table=default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.871299  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474cec4459954e10a18de031946a2346. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.872584  1234 ts_tablet_manager.cc:1428] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.000s
I20250628 07:52:12.876405  1234 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:12.882553  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 1642348a16574345b31cebda23575102 (DEFAULT_TABLE table=default.Table_a70f9c604ca44b7eae2b205bc80fdf1d [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.882920   802 tablet_service.cc:1468] Processing CreateTablet for tablet 1642348a16574345b31cebda23575102 (DEFAULT_TABLE table=default.Table_a70f9c604ca44b7eae2b205bc80fdf1d [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.883841  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1642348a16574345b31cebda23575102. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.883849   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1642348a16574345b31cebda23575102. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.883956  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.884855  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2df135033ee546f8bb9f4c4b54755b06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.885414   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 80f812df4f15406ebe0311c19bb654a7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:10 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.886732   800 tablet_service.cc:1468] Processing CreateTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.893656   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2df135033ee546f8bb9f4c4b54755b06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.894642   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:12.896975  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:12.899027   966 tablet_service.cc:1468] Processing CreateTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 (DEFAULT_TABLE table=default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.899688   965 tablet_service.cc:1468] Processing CreateTablet for tablet 474cec4459954e10a18de031946a2346 (DEFAULT_TABLE table=default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.899888   964 tablet_service.cc:1468] Processing CreateTablet for tablet 1642348a16574345b31cebda23575102 (DEFAULT_TABLE table=default.Table_a70f9c604ca44b7eae2b205bc80fdf1d [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.899943   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67a81eb4342740e5bc16ee7f5eacb489. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.900902   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1642348a16574345b31cebda23575102. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.902662   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:12.909842   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:12.900569   963 tablet_service.cc:1468] Processing CreateTablet for tablet 0f22700fc70d4f14ae51a834900db22b (DEFAULT_TABLE table=default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.911465   962 tablet_service.cc:1468] Processing CreateTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:12.917352   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 474cec4459954e10a18de031946a2346. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.921351   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2df135033ee546f8bb9f4c4b54755b06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.922547   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f22700fc70d4f14ae51a834900db22b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:12.925707  1234 tablet_bootstrap.cc:654] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:12.931138   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9863b171d29f48d88a2a547726aa8543" } new_table_name: "default.Table_2f8967092ad94cb9a5b4b41f44684f30" modify_external_catalogs: true
I20250628 07:52:12.936779   985 raft_consensus.cc:2466] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:12.943691  1062 leader_election.cc:304] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:12.944977  1280 raft_consensus.cc:2802] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:12.945497  1280 raft_consensus.cc:695] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:12.946588  1280 consensus_queue.cc:237] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:12.949657  1244 ts_tablet_manager.cc:1918] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:12.950067  1244 log.cc:1199] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/b5f50ad0815b433287d36bb209bee50d
I20250628 07:52:12.950747  1244 ts_tablet_manager.cc:1939] T b5f50ad0815b433287d36bb209bee50d P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:12.961480   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_5b9c6a7df6f04b808eecd7ee86d425b4 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 1 got 0
I20250628 07:52:12.962476  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "82865a5c5fc04d82a6be4c1d60fe9d7e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:12.963188  1129 raft_consensus.cc:3058] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:12.968331  1244 tablet_replica.cc:331] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:12.969152  1244 raft_consensus.cc:2241] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:12.968511   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet b5f50ad0815b433287d36bb209bee50d (table default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]) successfully deleted
I20250628 07:52:12.969702  1244 raft_consensus.cc:2270] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:12.974431  1129 raft_consensus.cc:2466] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:12.979321  1244 ts_tablet_manager.cc:1905] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.005573   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b7d509011ff1418c97d34d3e92013229"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.007257   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b7d509011ff1418c97d34d3e92013229 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.008275   921 leader_election.cc:304] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:13.009153   921 raft_consensus.cc:2684] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: unable to run election callback: Service unavailable: Thread pool token was shut down
I20250628 07:52:13.010807  1229 ts_tablet_manager.cc:1428] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.289s	user 0.016s	sys 0.013s
I20250628 07:52:13.011345  1237 leader_election.cc:290] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:13.013706  1245 ts_tablet_manager.cc:1905] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.015048  1246 ts_tablet_manager.cc:1918] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.015429  1246 log.cc:1199] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:13.016036  1246 ts_tablet_manager.cc:1939] T 80f812df4f15406ebe0311c19bb654a7 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.028148  1245 ts_tablet_manager.cc:1918] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
W20250628 07:52:13.029819   534 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 45896.4us
I20250628 07:52:13.033126   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_5b9c6a7df6f04b808eecd7ee86d425b4 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 1 got 0
I20250628 07:52:13.033221   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5b9c6a7df6f04b808eecd7ee86d425b4" } new_table_name: "default.Table_2952766630724aaaa5ed698e7c77fb35" modify_external_catalogs: true
I20250628 07:52:13.033030   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7f4db33aa37f41aaacb6a4e54f2189c3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.034711   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7f4db33aa37f41aaacb6a4e54f2189c3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.051930  1245 log.cc:1199] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/82865a5c5fc04d82a6be4c1d60fe9d7e
I20250628 07:52:13.029127  1229 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:13.057677  1245 ts_tablet_manager.cc:1939] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
W20250628 07:52:13.058149   543 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 consensus state has no leader
I20250628 07:52:13.060355  1229 tablet_bootstrap.cc:654] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.063301  1230 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.063957  1230 ts_tablet_manager.cc:1397] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.258s	user 0.013s	sys 0.030s
I20250628 07:52:13.065613  1234 tablet_bootstrap.cc:492] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:13.067035   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (table default.Table_fed6fabd13364e748240a44b749e4158 [id=0678e3fca14f4accbcf09e6ff76db055]) successfully deleted
I20250628 07:52:13.068893   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9be11b679e27417a8706dbdf185a798c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.070565   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9be11b679e27417a8706dbdf185a798c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:13.073237   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet ef449bd95fc64bf28c854b03a219c7b4 consensus state has no leader
I20250628 07:52:13.078898   541 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 7dc2190c92774207bdabeafb1298ab87
I20250628 07:52:13.089025  1234 ts_tablet_manager.cc:1397] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.213s	user 0.011s	sys 0.024s
W20250628 07:52:13.090379   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 67a81eb4342740e5bc16ee7f5eacb489 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 67a81eb4342740e5bc16ee7f5eacb489 already in progress: creating tablet
I20250628 07:52:13.099965   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1c78cba777ca43b6b253acb1d9c35a3a" } modify_external_catalogs: true
I20250628 07:52:13.100549   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1c78cba777ca43b6b253acb1d9c35a3a" } modify_external_catalogs: true
I20250628 07:52:13.110406  1244 ts_tablet_manager.cc:1918] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.110913  1244 log.cc:1199] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/80f812df4f15406ebe0311c19bb654a7
I20250628 07:52:13.111725  1244 ts_tablet_manager.cc:1939] T 80f812df4f15406ebe0311c19bb654a7 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:13.123462  1230 raft_consensus.cc:357] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.124076  1230 raft_consensus.cc:383] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.124274  1246 tablet_replica.cc:331] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:13.124310  1230 raft_consensus.cc:738] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.125036  1246 raft_consensus.cc:2241] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.124882  1230 consensus_queue.cc:260] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.125473  1246 raft_consensus.cc:2270] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.137293   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 80f812df4f15406ebe0311c19bb654a7 (table default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]) successfully deleted
I20250628 07:52:13.145196  1235 raft_consensus.cc:491] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.145622  1235 raft_consensus.cc:513] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.147387  1235 leader_election.cc:290] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:13.149255   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:13.149838   985 raft_consensus.cc:2466] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:13.151067  1062 leader_election.cc:304] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:13.151573  1230 ts_tablet_manager.cc:1428] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.087s	user 0.006s	sys 0.000s
I20250628 07:52:13.151947  1235 raft_consensus.cc:2802] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:13.152225  1235 raft_consensus.cc:491] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:13.152493  1235 raft_consensus.cc:3058] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:13.165705  1230 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.166451   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:13.108796  1234 raft_consensus.cc:357] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.170886  1234 raft_consensus.cc:383] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.171213  1234 raft_consensus.cc:738] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:13.171159  1062 leader_election.cc:343] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.171911  1234 consensus_queue.cc:260] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:13.177490   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20250628 07:52:13.178918   544 catalog_manager.cc:5582] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } }
I20250628 07:52:13.180280   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:13.181267   805 tablet_service.cc:1468] Processing CreateTablet for tablet fa96605dbabe4f0fbc26b057232960a4 (DEFAULT_TABLE table=default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.182559   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa96605dbabe4f0fbc26b057232960a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.182837  1246 ts_tablet_manager.cc:1905] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.185587   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 80f812df4f15406ebe0311c19bb654a7 (table default.Table_41a1bc0dc63041d1829de3ad0475bd56 [id=ea19eecfd5d0431b9facc9a5b5894d35]) successfully deleted
I20250628 07:52:13.188125  1230 tablet_bootstrap.cc:654] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.189843  1234 ts_tablet_manager.cc:1428] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.084s	user 0.006s	sys 0.001s
I20250628 07:52:13.192127   801 tablet_service.cc:1468] Processing CreateTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee (DEFAULT_TABLE table=default.Table_85234bd18f8e42a3b62dc723411c065a [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.195204   964 tablet_service.cc:1468] Processing CreateTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee (DEFAULT_TABLE table=default.Table_85234bd18f8e42a3b62dc723411c065a [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.196247   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83f8dcff296a4429a5f16bde59c8ebee. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:13.197460   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 67a81eb4342740e5bc16ee7f5eacb489 because the tablet was not found. No further retry: Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:13.199625   965 tablet_service.cc:1468] Processing CreateTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 (DEFAULT_TABLE table=default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2 [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.199970   799 tablet_service.cc:1468] Processing CreateTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 (DEFAULT_TABLE table=default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2 [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.200341   962 tablet_service.cc:1468] Processing CreateTablet for tablet fa96605dbabe4f0fbc26b057232960a4 (DEFAULT_TABLE table=default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.200608   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3223005021ae4e38b4ac65af460d0ad0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.201359   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa96605dbabe4f0fbc26b057232960a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.203008   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 3 got 0
I20250628 07:52:13.190451  1105 tablet_service.cc:1468] Processing CreateTablet for tablet fa96605dbabe4f0fbc26b057232960a4 (DEFAULT_TABLE table=default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.207667  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa96605dbabe4f0fbc26b057232960a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.214784  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 (DEFAULT_TABLE table=default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2 [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.215795  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3223005021ae4e38b4ac65af460d0ad0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.197808  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee (DEFAULT_TABLE table=default.Table_85234bd18f8e42a3b62dc723411c065a [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.217785  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83f8dcff296a4429a5f16bde59c8ebee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.217828   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3223005021ae4e38b4ac65af460d0ad0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.217967   798 tablet_service.cc:1468] Processing CreateTablet for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (DEFAULT_TABLE table=default.Table_26605bb2e3db42a78e439ffdd1f85944 [id=5e0bf38f7dc94bf59a2f39456ecae8fc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.219110   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 872ebf0393fa40cf82feebaad1ca3fa3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.220873   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 0f22700fc70d4f14ae51a834900db22b
I20250628 07:52:13.224062   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:13.225209   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:13.226621   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83f8dcff296a4429a5f16bde59c8ebee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.226797  1234 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:13.228971  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:13.232880  1235 raft_consensus.cc:513] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
W20250628 07:52:13.234104   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0f22700fc70d4f14ae51a834900db22b because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0f22700fc70d4f14ae51a834900db22b already in progress: creating tablet
I20250628 07:52:13.234812  1234 tablet_bootstrap.cc:654] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.237236   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 3 got 0
I20250628 07:52:13.241604  1229 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:13.242079  1229 ts_tablet_manager.cc:1397] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.213s	user 0.025s	sys 0.005s
I20250628 07:52:13.244796   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_58d9ee626715446c8457e182fc28efe2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.245957   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:13.246511   985 raft_consensus.cc:3058] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:13.246616   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_58d9ee626715446c8457e182fc28efe2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.250406  1229 raft_consensus.cc:357] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.252669  1229 raft_consensus.cc:383] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.254403  1235 leader_election.cc:290] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:13.254654  1229 raft_consensus.cc:738] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.253309   963 tablet_service.cc:1468] Processing CreateTablet for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (DEFAULT_TABLE table=default.Table_26605bb2e3db42a78e439ffdd1f85944 [id=5e0bf38f7dc94bf59a2f39456ecae8fc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.255304  1229 consensus_queue.cc:260] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.256174   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 872ebf0393fa40cf82feebaad1ca3fa3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.255123  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (DEFAULT_TABLE table=default.Table_26605bb2e3db42a78e439ffdd1f85944 [id=5e0bf38f7dc94bf59a2f39456ecae8fc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.259559  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 872ebf0393fa40cf82feebaad1ca3fa3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.264492   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1ce7bc7a0b5c4005b39592bf0d1b26c2" } new_table_name: "default.Table_b89abfdc7dcf4f8db19baf38a00d51ef" modify_external_catalogs: true
I20250628 07:52:13.254027   985 raft_consensus.cc:2466] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:13.265937  1062 leader_election.cc:304] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:13.267542  1280 raft_consensus.cc:2802] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:13.280246   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7f4db33aa37f41aaacb6a4e54f2189c3" } new_table_name: "default.Table_c84bc632fbed42de8d7f5bc303bcaa61" modify_external_catalogs: true
I20250628 07:52:13.285521   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_712625afc1034446a1f0bfd3618ea308" } modify_external_catalogs: true
I20250628 07:52:13.286082   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_712625afc1034446a1f0bfd3618ea308" } modify_external_catalogs: true
I20250628 07:52:13.287262  1230 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.287653  1230 ts_tablet_manager.cc:1397] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.122s	user 0.001s	sys 0.008s
I20250628 07:52:13.293527   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:13.294555  1062 leader_election.cc:343] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:13.298322   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0f22700fc70d4f14ae51a834900db22b because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0f22700fc70d4f14ae51a834900db22b already in progress: creating tablet
I20250628 07:52:13.299675  1229 ts_tablet_manager.cc:1428] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.057s	user 0.004s	sys 0.001s
I20250628 07:52:13.301507   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_75ab76ab0858432dbdf5873e82231726"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.301887   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7435230a3ac0447b942eb051eb09f728"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.302135   966 tablet_service.cc:1468] Processing CreateTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 (DEFAULT_TABLE table=default.Table_5455ab227ebe42608fed93cfb1b31b5a [id=3602ce86ce23434d83a8641b502843c4]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:13.303224   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7435230a3ac0447b942eb051eb09f728 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.303536   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2b51d6076e344c8f8e388b77b4b23928. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.305055   965 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:13.305733  1245 tablet_replica.cc:331] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:13.306340  1245 raft_consensus.cc:2241] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.306739  1245 raft_consensus.cc:2270] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.308184  1245 ts_tablet_manager.cc:1905] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:13.311357   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 3223005021ae4e38b4ac65af460d0ad0 consensus state has no leader
I20250628 07:52:13.316121  1310 raft_consensus.cc:491] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.316514  1310 raft_consensus.cc:513] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:13.303210   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_75ab76ab0858432dbdf5873e82231726 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.319603   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:13.320780  1062 leader_election.cc:343] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.322521  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 (DEFAULT_TABLE table=default.Table_5455ab227ebe42608fed93cfb1b31b5a [id=3602ce86ce23434d83a8641b502843c4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.323464  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2b51d6076e344c8f8e388b77b4b23928. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.300962   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bcf5c3bfda174712bc8205e0bbbfe24b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.324887   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bcf5c3bfda174712bc8205e0bbbfe24b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.325279   802 tablet_service.cc:1468] Processing CreateTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 (DEFAULT_TABLE table=default.Table_5455ab227ebe42608fed93cfb1b31b5a [id=3602ce86ce23434d83a8641b502843c4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.325981   965 tablet_service.cc:1468] Processing CreateTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (DEFAULT_TABLE table=default.Table_de1b1ac5b4e44da284d4cf266fef4876 [id=4d5813f4497543018828699e64223907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.327162   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.330499  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (DEFAULT_TABLE table=default.Table_de1b1ac5b4e44da284d4cf266fef4876 [id=4d5813f4497543018828699e64223907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.331769  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.333017  1230 raft_consensus.cc:357] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.333638  1230 raft_consensus.cc:383] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.333899  1230 raft_consensus.cc:738] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:13.345691   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1c65fa809b17418c85d4d72ae1e0ee3a has no consensus state
I20250628 07:52:13.346782   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet deb5a9ae7ab5463cae35ff1086fd8348
I20250628 07:52:13.350107  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:13.353348   961 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:13.355813  1280 raft_consensus.cc:695] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:13.356555  1280 consensus_queue.cc:237] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.358264  1309 raft_consensus.cc:491] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.358759  1309 raft_consensus.cc:513] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.360740  1245 ts_tablet_manager.cc:1918] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.366782  1229 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:13.368582  1230 consensus_queue.cc:260] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.370613  1230 ts_tablet_manager.cc:1428] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.083s	user 0.006s	sys 0.000s
I20250628 07:52:13.371419  1230 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.373145   803 tablet_service.cc:1468] Processing CreateTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (DEFAULT_TABLE table=default.Table_de1b1ac5b4e44da284d4cf266fef4876 [id=4d5813f4497543018828699e64223907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.374917   800 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:13.375895   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:13.376797  1245 log.cc:1199] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/b8b1e09d0f224c10ab8df26f12239254
I20250628 07:52:13.377236   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.377691  1245 ts_tablet_manager.cc:1939] T b8b1e09d0f224c10ab8df26f12239254 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:13.380158   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet b8b1e09d0f224c10ab8df26f12239254 (table default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]) successfully deleted
I20250628 07:52:13.381472   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2b51d6076e344c8f8e388b77b4b23928. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:13.385514  1062 leader_election.cc:343] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.385934  1062 leader_election.cc:304] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:13.386796  1235 raft_consensus.cc:2747] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:13.403249  1229 tablet_bootstrap.cc:654] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:13.398150   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet deb5a9ae7ab5463cae35ff1086fd8348 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet deb5a9ae7ab5463cae35ff1086fd8348 already in progress: creating tablet
I20250628 07:52:13.393047   547 catalog_manager.cc:5582] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:13.405494   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_2952766630724aaaa5ed698e7c77fb35 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 2 got 0
I20250628 07:52:13.410120  1246 ts_tablet_manager.cc:1918] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.410526  1246 log.cc:1199] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/b5f50ad0815b433287d36bb209bee50d
I20250628 07:52:13.411279  1246 ts_tablet_manager.cc:1939] T b5f50ad0815b433287d36bb209bee50d P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.418884  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 32) took 1230 ms. Trace:
I20250628 07:52:13.419246  1246 rpcz_store.cc:276] 0628 07:52:12.188513 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.278716 (+ 90203us) service_pool.cc:225] Handling call
0628 07:52:13.124122 (+845406us) ts_tablet_manager.cc:1145] Deleting tablet b5f50ad0815b433287d36bb209bee50d
0628 07:52:13.124166 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.214407 (+ 90241us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.409879 (+195472us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.418820 (+  8941us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":55410,"tablet-delete.queue_time_us":844616}
I20250628 07:52:13.420914   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet b5f50ad0815b433287d36bb209bee50d (table default.Table_5d03e239a7ab48779e36a906630751bd [id=8296732ecf664ccea39bf6fc9d09f21d]) successfully deleted
I20250628 07:52:13.421511  1230 tablet_bootstrap.cc:654] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.319409  1310 leader_election.cc:290] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:13.424810  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 37) took 1072 ms. Trace:
I20250628 07:52:13.426216  1246 rpcz_store.cc:276] 0628 07:52:12.352944 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.353109 (+   165us) service_pool.cc:225] Handling call
0628 07:52:13.424455 (+1071346us) ts_tablet_manager.cc:1145] Deleting tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67
0628 07:52:13.424492 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.424752 (+   260us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1070670}
W20250628 07:52:13.427394   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 already in progress: creating tablet
I20250628 07:52:13.428234  1246 tablet_replica.cc:331] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:13.428972  1246 raft_consensus.cc:2241] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.429534  1246 raft_consensus.cc:2270] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.431206  1246 ts_tablet_manager.cc:1905] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.438738   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7435230a3ac0447b942eb051eb09f728" } modify_external_catalogs: true
I20250628 07:52:13.439289   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7435230a3ac0447b942eb051eb09f728" } modify_external_catalogs: true
I20250628 07:52:13.441449  1230 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.441879  1230 ts_tablet_manager.cc:1397] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.071s	user 0.011s	sys 0.001s
I20250628 07:52:13.444000  1230 raft_consensus.cc:357] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.444573  1230 raft_consensus.cc:383] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.444830  1230 raft_consensus.cc:738] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.446743  1230 consensus_queue.cc:260] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.450979   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b8b1e09d0f224c10ab8df26f12239254" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:13.451275  1062 leader_election.cc:343] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: b8b1e09d0f224c10ab8df26f12239254
W20250628 07:52:13.453225  1062 leader_election.cc:343] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.453591  1062 leader_election.cc:304] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:13.455562  1280 raft_consensus.cc:2747] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:13.455628  1309 leader_election.cc:290] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
W20250628 07:52:13.457695   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet deb5a9ae7ab5463cae35ff1086fd8348 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet deb5a9ae7ab5463cae35ff1086fd8348 already in progress: creating tablet
I20250628 07:52:13.459973  1230 ts_tablet_manager.cc:1428] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.000s
I20250628 07:52:13.465318  1230 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.492341   544 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 90a2c9fd8f3147e4a688733510debf15
I20250628 07:52:13.496042  1230 tablet_bootstrap.cc:654] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.501777  1234 tablet_bootstrap.cc:492] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:13.502235  1234 ts_tablet_manager.cc:1397] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.276s	user 0.011s	sys 0.007s
I20250628 07:52:13.503326  1229 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:13.503769  1229 ts_tablet_manager.cc:1397] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.137s	user 0.023s	sys 0.012s
I20250628 07:52:13.504052  1246 ts_tablet_manager.cc:1918] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.504458  1246 log.cc:1199] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/752fc37e7210458fb0c631d01204a3bb
I20250628 07:52:13.505465  1246 ts_tablet_manager.cc:1939] T 752fc37e7210458fb0c631d01204a3bb P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.506618  1229 raft_consensus.cc:357] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:13.507302  1229 raft_consensus.cc:383] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.507588  1229 raft_consensus.cc:738] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.510185  1234 raft_consensus.cc:357] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.510830  1234 raft_consensus.cc:383] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.511103  1234 raft_consensus.cc:738] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.511138   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.511040  1229 consensus_queue.cc:260] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
W20250628 07:52:13.512754   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.514590   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5455ab227ebe42608fed93cfb1b31b5a" } new_table_name: "default.Table_c22b7ab2180346e093151880dbd4cb83" modify_external_catalogs: true
I20250628 07:52:13.518860  1234 consensus_queue.cc:260] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.523298  1229 ts_tablet_manager.cc:1428] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.019s	user 0.005s	sys 0.000s
I20250628 07:52:13.524446  1234 ts_tablet_manager.cc:1428] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.022s	user 0.007s	sys 0.001s
I20250628 07:52:13.525492  1234 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:13.528211   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:13.532131   962 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:13.534058  1245 tablet_replica.cc:331] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:13.534644  1245 raft_consensus.cc:2241] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.535029  1245 raft_consensus.cc:2270] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.535516  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 38) took 1106 ms. Trace:
I20250628 07:52:13.535851  1246 rpcz_store.cc:276] 0628 07:52:12.429174 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.429287 (+   113us) service_pool.cc:225] Handling call
0628 07:52:13.428092 (+998805us) ts_tablet_manager.cc:1145] Deleting tablet 752fc37e7210458fb0c631d01204a3bb
0628 07:52:13.428130 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.487597 (+ 59467us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.503823 (+ 16226us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.535450 (+ 31627us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":998311}
I20250628 07:52:13.537449  1245 ts_tablet_manager.cc:1905] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.539090   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:13.543131   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 752fc37e7210458fb0c631d01204a3bb (table default.Table_9c39cbe761084240846a9147b9666d33 [id=4c6801aa5535410980af48c4ca4eb5d5]) successfully deleted
I20250628 07:52:13.544003  1246 tablet_replica.cc:331] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:13.546314  1246 raft_consensus.cc:2241] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.546774  1246 raft_consensus.cc:2270] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.548300  1246 ts_tablet_manager.cc:1905] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.550276  1229 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:13.553777   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d2c7ee9d1fa644da9621c878e540a265" } new_table_name: "default.Table_2d6aec0ecd124912b4fdcea46bff2243" modify_external_catalogs: true
I20250628 07:52:13.554404   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_344e153bf04b4bf19e388f9f092ce6fb" } modify_external_catalogs: true
I20250628 07:52:13.554937   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_344e153bf04b4bf19e388f9f092ce6fb" } modify_external_catalogs: true
I20250628 07:52:13.557394  1230 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.557834  1230 ts_tablet_manager.cc:1397] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.093s	user 0.007s	sys 0.007s
I20250628 07:52:13.559463  1234 tablet_bootstrap.cc:654] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.560953  1230 raft_consensus.cc:357] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.561579  1230 raft_consensus.cc:383] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.561873  1230 raft_consensus.cc:738] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.563495  1230 consensus_queue.cc:260] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.577839  1229 tablet_bootstrap.cc:654] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.578697   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]). Expected version 1 got 0
I20250628 07:52:13.584180  1107 tablet_service.cc:1468] Processing CreateTablet for tablet c84f2570b2da45d0b3ce6428723263a5 (DEFAULT_TABLE table=default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.584584  1103 tablet_service.cc:1468] Processing CreateTablet for tablet d39d14572c4647c7b4880c71a8707c08 (DEFAULT_TABLE table=default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.585343  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84f2570b2da45d0b3ce6428723263a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.585878  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d39d14572c4647c7b4880c71a8707c08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.586357   802 tablet_service.cc:1468] Processing CreateTablet for tablet c84f2570b2da45d0b3ce6428723263a5 (DEFAULT_TABLE table=default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.587019   798 tablet_service.cc:1468] Processing CreateTablet for tablet d39d14572c4647c7b4880c71a8707c08 (DEFAULT_TABLE table=default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.587371   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84f2570b2da45d0b3ce6428723263a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.587962   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d39d14572c4647c7b4880c71a8707c08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.588729   803 tablet_service.cc:1468] Processing CreateTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (DEFAULT_TABLE table=default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.589607   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1c65fa809b17418c85d4d72ae1e0ee3a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.590998   962 tablet_service.cc:1468] Processing CreateTablet for tablet d39d14572c4647c7b4880c71a8707c08 (DEFAULT_TABLE table=default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.592104   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d39d14572c4647c7b4880c71a8707c08. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.592111   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_2e20136b218e40b6a54ab31e306e147e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.592993   543 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2b51d6076e344c8f8e388b77b4b23928 consensus state has no leader
W20250628 07:52:13.593796   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_2e20136b218e40b6a54ab31e306e147e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.593744   966 tablet_service.cc:1468] Processing CreateTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (DEFAULT_TABLE table=default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.594611   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1c65fa809b17418c85d4d72ae1e0ee3a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.603183  1310 raft_consensus.cc:491] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.603672  1310 raft_consensus.cc:513] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.605497  1310 leader_election.cc:290] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:13.607257   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:13.609148  1062 leader_election.cc:343] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.611714   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:13.613009  1062 leader_election.cc:343] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.613410  1062 leader_election.cc:304] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:13.615561  1310 raft_consensus.cc:2747] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:13.617626  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (DEFAULT_TABLE table=default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.618346   801 tablet_service.cc:1468] Processing CreateTablet for tablet e28ab08f878d4d4b86a5646f687b08fc (DEFAULT_TABLE table=default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2 [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.618671  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1c65fa809b17418c85d4d72ae1e0ee3a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.619336   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e28ab08f878d4d4b86a5646f687b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.620556  1245 ts_tablet_manager.cc:1918] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.620705  1108 tablet_service.cc:1468] Processing CreateTablet for tablet e28ab08f878d4d4b86a5646f687b08fc (DEFAULT_TABLE table=default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2 [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.620890  1245 log.cc:1199] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c1a06bbe216f419598a587056f9b8d05
I20250628 07:52:13.621569  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e28ab08f878d4d4b86a5646f687b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.621642   965 tablet_service.cc:1468] Processing CreateTablet for tablet c84f2570b2da45d0b3ce6428723263a5 (DEFAULT_TABLE table=default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.621763   964 tablet_service.cc:1468] Processing CreateTablet for tablet e28ab08f878d4d4b86a5646f687b08fc (DEFAULT_TABLE table=default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2 [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.622537   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84f2570b2da45d0b3ce6428723263a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.623342   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e28ab08f878d4d4b86a5646f687b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.624953   963 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (DEFAULT_TABLE table=default.Table_58d9ee626715446c8457e182fc28efe2 [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.625882   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.625711   805 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (DEFAULT_TABLE table=default.Table_58d9ee626715446c8457e182fc28efe2 [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.626569   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.629784  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (DEFAULT_TABLE table=default.Table_58d9ee626715446c8457e182fc28efe2 [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.631263  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.637228  1230 ts_tablet_manager.cc:1428] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.079s	user 0.005s	sys 0.001s
I20250628 07:52:13.646144   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9045e51711844332acd5d748d0bbc3af"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.647722   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9045e51711844332acd5d748d0bbc3af in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.648890  1245 ts_tablet_manager.cc:1939] T c1a06bbe216f419598a587056f9b8d05 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:13.651274   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet c1a06bbe216f419598a587056f9b8d05 (table default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]) successfully deleted
I20250628 07:52:13.652796  1230 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.657379  1229 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:13.657759  1229 ts_tablet_manager.cc:1397] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.108s	user 0.004s	sys 0.006s
I20250628 07:52:13.660346  1229 raft_consensus.cc:357] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.661643  1230 tablet_bootstrap.cc:654] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.667341  1234 tablet_bootstrap.cc:492] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:13.667714  1234 ts_tablet_manager.cc:1397] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.142s	user 0.004s	sys 0.005s
W20250628 07:52:13.669667   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 1c65fa809b17418c85d4d72ae1e0ee3a consensus state has no leader
I20250628 07:52:13.669725  1246 ts_tablet_manager.cc:1918] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.670090  1246 log.cc:1199] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/c1a06bbe216f419598a587056f9b8d05
I20250628 07:52:13.670540  1234 raft_consensus.cc:357] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.671190  1234 raft_consensus.cc:383] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.671438  1234 raft_consensus.cc:738] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.672642  1246 ts_tablet_manager.cc:1939] T c1a06bbe216f419598a587056f9b8d05 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.679435   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 474cec4459954e10a18de031946a2346
I20250628 07:52:13.680034   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c8d2155d54dc47c0a639ace22ba757eb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.681619   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c8d2155d54dc47c0a639ace22ba757eb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.683405   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:13.684595  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:13.685117   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:13.686307   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 474cec4459954e10a18de031946a2346 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 474cec4459954e10a18de031946a2346 already in progress: creating tablet
W20250628 07:52:13.687016   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 474cec4459954e10a18de031946a2346 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 474cec4459954e10a18de031946a2346 already in progress: creating tablet
I20250628 07:52:13.660946  1229 raft_consensus.cc:383] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.689771  1229 raft_consensus.cc:738] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
W20250628 07:52:13.690897   541 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 3637ab35a2b74ce3a111c77cbec8b0b0 consensus state has no leader
I20250628 07:52:13.693020  1230 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.694939  1230 ts_tablet_manager.cc:1397] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.042s	user 0.007s	sys 0.002s
I20250628 07:52:13.696883  1230 raft_consensus.cc:357] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.697525  1230 raft_consensus.cc:383] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.697522   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_75ab76ab0858432dbdf5873e82231726" } modify_external_catalogs: true
I20250628 07:52:13.697338  1229 consensus_queue.cc:260] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.697743  1230 raft_consensus.cc:738] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.697970   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_75ab76ab0858432dbdf5873e82231726" } modify_external_catalogs: true
I20250628 07:52:13.698272  1230 consensus_queue.cc:260] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.699170  1229 ts_tablet_manager.cc:1428] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.041s	user 0.005s	sys 0.000s
I20250628 07:52:13.699605  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 41) took 1174 ms. Trace:
I20250628 07:52:13.699959  1229 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:13.699877  1246 rpcz_store.cc:276] 0628 07:52:12.525206 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.525326 (+   120us) service_pool.cc:225] Handling call
0628 07:52:13.543864 (+1018538us) ts_tablet_manager.cc:1145] Deleting tablet c1a06bbe216f419598a587056f9b8d05
0628 07:52:13.543903 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.656684 (+112781us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.669530 (+ 12846us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.699388 (+ 29858us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":1639,"tablet-delete.queue_time_us":1017944}
I20250628 07:52:13.700996   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet c1a06bbe216f419598a587056f9b8d05 (table default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]) successfully deleted
I20250628 07:52:13.701789  1246 tablet_replica.cc:331] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:13.702404  1246 raft_consensus.cc:2241] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.712211   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:13.720623  1230 ts_tablet_manager.cc:1428] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.025s	user 0.004s	sys 0.000s
I20250628 07:52:13.722801  1246 raft_consensus.cc:2270] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.724412  1246 ts_tablet_manager.cc:1905] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.725353  1234 consensus_queue.cc:260] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.738263  1230 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.738876  1234 ts_tablet_manager.cc:1428] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.071s	user 0.005s	sys 0.001s
I20250628 07:52:13.743096  1234 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:13.756469  1229 tablet_bootstrap.cc:654] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.770015   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 360d3b0ffaec462aa9bd422f34269b66
I20250628 07:52:13.770493  1310 consensus_queue.cc:1035] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:13.770969  1230 tablet_bootstrap.cc:654] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.775257  1246 ts_tablet_manager.cc:1918] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.775640  1246 log.cc:1199] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/82865a5c5fc04d82a6be4c1d60fe9d7e
I20250628 07:52:13.776324  1246 ts_tablet_manager.cc:1939] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.778255  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 44) took 1121 ms. Trace:
I20250628 07:52:13.778591  1246 rpcz_store.cc:276] 0628 07:52:12.656532 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.656651 (+   119us) service_pool.cc:225] Handling call
0628 07:52:13.701658 (+1045007us) ts_tablet_manager.cc:1145] Deleting tablet 82865a5c5fc04d82a6be4c1d60fe9d7e
0628 07:52:13.701697 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.762939 (+ 61242us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.774976 (+ 12037us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.778185 (+  3209us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":18410,"tablet-delete.queue_time_us":1044262}
I20250628 07:52:13.781360  1246 tablet_replica.cc:331] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:13.781581  1237 raft_consensus.cc:491] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.782019  1246 raft_consensus.cc:2241] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.782017  1237 raft_consensus.cc:513] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.782384  1246 raft_consensus.cc:2270] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.783704  1237 leader_election.cc:290] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:13.783854  1246 ts_tablet_manager.cc:1905] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.784647  1234 tablet_bootstrap.cc:654] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.788338   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (table default.Table_fed6fabd13364e748240a44b749e4158 [id=0678e3fca14f4accbcf09e6ff76db055]) successfully deleted
I20250628 07:52:13.799571  1229 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:13.800143  1229 ts_tablet_manager.cc:1397] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.100s	user 0.012s	sys 0.000s
I20250628 07:52:13.800072   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_26605bb2e3db42a78e439ffdd1f85944" } new_table_name: "default.Table_a0e49295ed0e482eaa16a7761b01b32c" modify_external_catalogs: true
I20250628 07:52:13.802834  1229 raft_consensus.cc:357] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.803576  1229 raft_consensus.cc:383] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.803874  1229 raft_consensus.cc:738] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.804548  1229 consensus_queue.cc:260] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.805925   801 tablet_service.cc:1515] Processing DeleteTablet for tablet fbd97bd5efec4a989374c6c08e312e08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:13.807317  1244 tablet_replica.cc:331] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:13.808120  1244 raft_consensus.cc:2241] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.808542  1244 raft_consensus.cc:2270] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.809410   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:13.810468   920 leader_election.cc:343] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.812394  1244 ts_tablet_manager.cc:1905] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.818537  1229 ts_tablet_manager.cc:1428] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.000s
I20250628 07:52:13.819445  1229 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:13.827251  1229 tablet_bootstrap.cc:654] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:13.832926   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 872ebf0393fa40cf82feebaad1ca3fa3 consensus state has no leader
I20250628 07:52:13.835937  1246 ts_tablet_manager.cc:1918] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:13.836270  1246 log.cc:1199] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/b8b1e09d0f224c10ab8df26f12239254
I20250628 07:52:13.836855  1246 ts_tablet_manager.cc:1939] T b8b1e09d0f224c10ab8df26f12239254 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:13.838004   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:13.843575  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:13.844128  1129 raft_consensus.cc:2466] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:13.851434  1246 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 45) took 1192 ms. Trace:
I20250628 07:52:13.851826  1246 rpcz_store.cc:276] 0628 07:52:12.658800 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:12.658956 (+   156us) service_pool.cc:225] Handling call
0628 07:52:13.781191 (+1122235us) ts_tablet_manager.cc:1145] Deleting tablet b8b1e09d0f224c10ab8df26f12239254
0628 07:52:13.781245 (+    54us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:13.828867 (+ 47622us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.835741 (+  6874us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:13.851378 (+ 15637us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":3166,"tablet-delete.queue_time_us":1121655}
I20250628 07:52:13.852253  1108 tablet_service.cc:1468] Processing CreateTablet for tablet de7ff3260b0049d682dba82f914474de (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:13.853140  1062 consensus_peers.cc:489] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:13.853457  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet de7ff3260b0049d682dba82f914474de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.853377   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet b8b1e09d0f224c10ab8df26f12239254 (table default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]) successfully deleted
I20250628 07:52:13.841467  1244 ts_tablet_manager.cc:1918] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
W20250628 07:52:13.854440   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 67a81eb4342740e5bc16ee7f5eacb489 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 67a81eb4342740e5bc16ee7f5eacb489 already in progress: creating tablet
W20250628 07:52:13.855530   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0f22700fc70d4f14ae51a834900db22b because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0f22700fc70d4f14ae51a834900db22b already in progress: creating tablet
W20250628 07:52:13.856122   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet deb5a9ae7ab5463cae35ff1086fd8348 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet deb5a9ae7ab5463cae35ff1086fd8348 already in progress: creating tablet
W20250628 07:52:13.857467   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 474cec4459954e10a18de031946a2346 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 474cec4459954e10a18de031946a2346 already in progress: creating tablet
I20250628 07:52:13.859249  1230 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:13.859653  1230 ts_tablet_manager.cc:1397] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.122s	user 0.010s	sys 0.005s
I20250628 07:52:13.841905  1292 raft_consensus.cc:491] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.851053   801 tablet_service.cc:1468] Processing CreateTablet for tablet de7ff3260b0049d682dba82f914474de (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.865273   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet de7ff3260b0049d682dba82f914474de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.864784  1292 raft_consensus.cc:513] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.854521   802 tablet_service.cc:1468] Processing CreateTablet for tablet 1eac71e98ae44f41bb17816618c7d897 (DEFAULT_TABLE table=default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.869730   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1eac71e98ae44f41bb17816618c7d897. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.871412   963 tablet_service.cc:1468] Processing CreateTablet for tablet de7ff3260b0049d682dba82f914474de (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:13.872411   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet de7ff3260b0049d682dba82f914474de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:13.874389  1292 leader_election.cc:290] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:13.864284  1234 tablet_bootstrap.cc:492] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:13.876101  1234 ts_tablet_manager.cc:1397] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.133s	user 0.005s	sys 0.005s
I20250628 07:52:13.879097   921 leader_election.cc:304] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:13.882586   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_218e7d52d78541a0b85a22af790324a6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.884006   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_218e7d52d78541a0b85a22af790324a6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.884157  1237 raft_consensus.cc:2802] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:13.884518  1237 raft_consensus.cc:491] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:13.884878  1237 raft_consensus.cc:3058] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:13.888782   985 raft_consensus.cc:1273] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Refusing update from remote peer 6a58514e01ea4989809bc2992fb2b806: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:13.892663  1244 log.cc:1199] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/fbd97bd5efec4a989374c6c08e312e08
I20250628 07:52:13.893991  1244 ts_tablet_manager.cc:1939] T fbd97bd5efec4a989374c6c08e312e08 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:13.896840   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_eb3cdbf1d68d4a2c9fa009c2c1c3e6d2" } new_table_name: "default.Table_8146958ca541498babe752911c4346bd" modify_external_catalogs: true
I20250628 07:52:13.897549   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c22b7ab2180346e093151880dbd4cb83" } modify_external_catalogs: true
I20250628 07:52:13.898121   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c22b7ab2180346e093151880dbd4cb83" } modify_external_catalogs: true
I20250628 07:52:13.909449   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet fbd97bd5efec4a989374c6c08e312e08 (table default.Table_2770a34952aa436fb1566f2efc41b514 [id=2c1addd827ca462289c3e5ec95fe2bcd]) successfully deleted
I20250628 07:52:13.911888  1237 raft_consensus.cc:513] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.914608   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:13.916523   920 leader_election.cc:343] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:13.921741  1234 raft_consensus.cc:357] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.922339  1234 raft_consensus.cc:383] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.922549  1234 raft_consensus.cc:738] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.923453  1234 consensus_queue.cc:260] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.940416  1234 ts_tablet_manager.cc:1428] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.064s	user 0.004s	sys 0.000s
I20250628 07:52:13.940835   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:13.941262  1234 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:13.946768  1230 raft_consensus.cc:357] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.947399  1230 raft_consensus.cc:383] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:13.947639  1230 raft_consensus.cc:738] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:13.948258  1230 consensus_queue.cc:260] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.950860  1230 ts_tablet_manager.cc:1428] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.091s	user 0.005s	sys 0.000s
W20250628 07:52:13.961141   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet e28ab08f878d4d4b86a5646f687b08fc consensus state has no leader
I20250628 07:52:13.961855   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1612b9b2623a4d708b4af562bed58f69"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.962906   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (table default.Table_2d6aec0ecd124912b4fdcea46bff2243 [id=853998210b39471f9d2d0bf0ebf072d5]). Expected version 1 got 0
W20250628 07:52:13.963517   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1612b9b2623a4d708b4af562bed58f69 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.973742  1230 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:13.973126   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_177b8783c82149aabfddcfe195d451a7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:13.975579   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_374f5d5ed8ee4ad481b7071e0d2865e1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:13.978502   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_374f5d5ed8ee4ad481b7071e0d2865e1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.979315  1230 tablet_bootstrap.cc:654] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:13.982774  1234 tablet_bootstrap.cc:654] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:13.983654   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_177b8783c82149aabfddcfe195d451a7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:13.984004  1235 raft_consensus.cc:491] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:13.984437  1235 raft_consensus.cc:513] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:13.986163  1235 leader_election.cc:290] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:13.990609   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9be11b679e27417a8706dbdf185a798c" } modify_external_catalogs: true
I20250628 07:52:13.991192   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9be11b679e27417a8706dbdf185a798c" } modify_external_catalogs: true
I20250628 07:52:13.994174   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 82865a5c5fc04d82a6be4c1d60fe9d7e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:13.995504  1244 tablet_replica.cc:331] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:13.996248  1244 raft_consensus.cc:2241] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:13.996686  1244 raft_consensus.cc:2270] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:13.996840   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:13.998740  1244 ts_tablet_manager.cc:1905] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:13.999781  1330 raft_consensus.cc:491] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.015514  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 1eac71e98ae44f41bb17816618c7d897 (DEFAULT_TABLE table=default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.017153  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1eac71e98ae44f41bb17816618c7d897. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.019421   825 raft_consensus.cc:3058] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.022583  1234 tablet_bootstrap.cc:492] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.023010  1234 ts_tablet_manager.cc:1397] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.082s	user 0.006s	sys 0.003s
I20250628 07:52:14.025064  1234 raft_consensus.cc:357] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.025691  1234 raft_consensus.cc:383] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.025934  1234 raft_consensus.cc:738] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.026475  1234 consensus_queue.cc:260] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.028880   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:14.033578  1235 consensus_queue.cc:1035] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:14.034148  1062 leader_election.cc:343] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.035598  1235 consensus_queue.cc:1035] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:14.039433  1234 ts_tablet_manager.cc:1428] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.016s	user 0.001s	sys 0.005s
I20250628 07:52:14.040287   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_18d9b7c303dd43c68262a4c621f81364" } new_table_name: "default.Table_b4f083a990fb4f3493ed3a44c9fb3fac" modify_external_catalogs: true
I20250628 07:52:14.045341  1230 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.045733  1230 ts_tablet_manager.cc:1397] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.072s	user 0.009s	sys 0.000s
I20250628 07:52:14.048801   801 tablet_service.cc:1515] Processing DeleteTablet for tablet c1a06bbe216f419598a587056f9b8d05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:14.000237  1330 raft_consensus.cc:513] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.053002  1234 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.062131  1244 ts_tablet_manager.cc:1918] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:14.062547  1244 log.cc:1199] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/82865a5c5fc04d82a6be4c1d60fe9d7e
I20250628 07:52:14.063162  1244 ts_tablet_manager.cc:1939] T 82865a5c5fc04d82a6be4c1d60fe9d7e P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:14.067914  1230 raft_consensus.cc:357] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.068562  1230 raft_consensus.cc:383] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.068802  1230 raft_consensus.cc:738] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.074561  1330 leader_election.cc:290] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.076220  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:14.076826  1129 raft_consensus.cc:2466] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:14.078573   921 leader_election.cc:304] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:14.079445  1330 raft_consensus.cc:2802] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:14.079787  1330 raft_consensus.cc:491] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:14.080119  1330 raft_consensus.cc:3058] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.088958   965 tablet_service.cc:1468] Processing CreateTablet for tablet 1eac71e98ae44f41bb17816618c7d897 (DEFAULT_TABLE table=default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.093644   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:14.097149  1062 leader_election.cc:343] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.097570  1062 leader_election.cc:304] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.098562  1280 raft_consensus.cc:2747] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.092449  1237 leader_election.cc:290] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.100351   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1eac71e98ae44f41bb17816618c7d897. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.106369   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:14.109820   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:14.111788   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2b51d6076e344c8f8e388b77b4b23928 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2b51d6076e344c8f8e388b77b4b23928 already in progress: creating tablet
I20250628 07:52:14.112713  1229 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:14.113118  1229 ts_tablet_manager.cc:1397] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.294s	user 0.012s	sys 0.000s
I20250628 07:52:14.116783  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.117584  1246 tablet_replica.cc:331] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:14.118245  1246 raft_consensus.cc:2241] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:14.118664  1246 raft_consensus.cc:2270] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:14.119050  1229 raft_consensus.cc:357] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.119685  1229 raft_consensus.cc:383] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.119937  1229 raft_consensus.cc:738] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.121608  1246 ts_tablet_manager.cc:1905] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:14.122619  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:14.123118  1129 raft_consensus.cc:3058] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.125130  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.130465  1341 tablet.cc:1722] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:14.168448  1342 tablet.cc:1722] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:14.175547  1229 consensus_queue.cc:260] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.077910  1234 tablet_bootstrap.cc:654] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.180502   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 82865a5c5fc04d82a6be4c1d60fe9d7e (table default.Table_fed6fabd13364e748240a44b749e4158 [id=0678e3fca14f4accbcf09e6ff76db055]) successfully deleted
I20250628 07:52:14.078150   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:14.108088   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:14.184566   920 leader_election.cc:343] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.185667  1237 raft_consensus.cc:491] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.186164  1237 raft_consensus.cc:513] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.188925  1330 raft_consensus.cc:513] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.193455  1330 leader_election.cc:290] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.194862  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:14.195465  1128 raft_consensus.cc:3058] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.198421   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:14.202384  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.202957   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:14.203470   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_58d9ee626715446c8457e182fc28efe2" } new_table_name: "default.Table_21bdd4d912164bcbaf63188ced2ce8ad" modify_external_catalogs: true
I20250628 07:52:14.204447  1128 raft_consensus.cc:2466] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:14.205842   920 leader_election.cc:343] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.204516   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_54b2cdede05f442cb652727e32c46009"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:14.206513   921 leader_election.cc:304] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:14.207299   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_54b2cdede05f442cb652727e32c46009 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.209065  1229 ts_tablet_manager.cc:1428] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.096s	user 0.005s	sys 0.000s
I20250628 07:52:14.210734  1229 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:14.211447  1234 tablet_bootstrap.cc:492] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.214437  1234 ts_tablet_manager.cc:1397] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.162s	user 0.007s	sys 0.011s
I20250628 07:52:14.216495  1330 raft_consensus.cc:2802] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:14.216579  1234 raft_consensus.cc:357] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.217206  1234 raft_consensus.cc:383] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.217286  1330 raft_consensus.cc:695] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:14.217438  1234 raft_consensus.cc:738] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.218199  1330 consensus_queue.cc:237] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.225471  1234 consensus_queue.cc:260] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.238700  1234 ts_tablet_manager.cc:1428] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.024s	user 0.005s	sys 0.000s
I20250628 07:52:14.239979  1234 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.245338  1234 tablet_bootstrap.cc:654] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.254186   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:14.256765   801 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:14.268501  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.275136  1234 tablet_bootstrap.cc:492] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.275748  1234 ts_tablet_manager.cc:1397] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.036s	user 0.003s	sys 0.006s
I20250628 07:52:14.279102  1234 raft_consensus.cc:357] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.279356  1230 consensus_queue.cc:260] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.280059  1234 raft_consensus.cc:383] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.280419  1234 raft_consensus.cc:738] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.287043  1230 ts_tablet_manager.cc:1428] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.241s	user 0.005s	sys 0.000s
I20250628 07:52:14.288532  1234 consensus_queue.cc:260] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.295424  1234 ts_tablet_manager.cc:1428] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.019s	user 0.008s	sys 0.000s
I20250628 07:52:14.296260  1234 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.300915  1129 raft_consensus.cc:2466] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:14.317147  1234 tablet_bootstrap.cc:654] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.318152   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b7d509011ff1418c97d34d3e92013229" } modify_external_catalogs: true
I20250628 07:52:14.318706   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b7d509011ff1418c97d34d3e92013229" } modify_external_catalogs: true
I20250628 07:52:14.329865  1244 tablet_replica.cc:331] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:14.330879  1244 raft_consensus.cc:2241] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:14.331472  1244 raft_consensus.cc:2270] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:14.333544  1244 ts_tablet_manager.cc:1905] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:14.339210  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63348a64bb7044dbad9065afff179fbc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:14.342526   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 consensus state has no leader
I20250628 07:52:14.347752   758 leader_election.cc:304] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.349545  1292 raft_consensus.cc:2747] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.349907   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_62272d797cf643d295305387e17a746b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:14.351440   824 raft_consensus.cc:3058] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:14.351601   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_62272d797cf643d295305387e17a746b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.359858   547 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]). Expected version 1 got 0
I20250628 07:52:14.369465   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8c096e0d11db417187a1bde167c0735f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:14.371217   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8c096e0d11db417187a1bde167c0735f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.387562   921 leader_election.cc:304] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:14.388495  1339 raft_consensus.cc:2802] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:14.396399   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:14.397941   920 leader_election.cc:343] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.399004   963 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:14.412787  1341 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:14.419339  1237 leader_election.cc:290] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.426273  1230 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:14.442852   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_18d9b7c303dd43c68262a4c621f81364 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 1 got 0
W20250628 07:52:14.443966   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2d77be1f37d2400f842c01019e4eb474 consensus state has no leader
I20250628 07:52:14.453235  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 (DEFAULT_TABLE table=default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.453037   801 tablet_service.cc:1468] Processing CreateTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 (DEFAULT_TABLE table=default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.454216  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4e9d585d5f0c4469828561a7df4fb917. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.455001   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4e9d585d5f0c4469828561a7df4fb917. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.455190   802 tablet_service.cc:1468] Processing CreateTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb (DEFAULT_TABLE table=default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.456117   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 950fea8f6bfb409f8868fe5e88d3dddb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.459415  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 8c1491faa1be485792b6193d4ca22f37 (DEFAULT_TABLE table=default.Table_2e20136b218e40b6a54ab31e306e147e [id=ca323f84b00048e69461d15b7bd6369a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.459523  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb (DEFAULT_TABLE table=default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.459586  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (DEFAULT_TABLE table=default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.460515  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8c1491faa1be485792b6193d4ca22f37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.460678   803 tablet_service.cc:1468] Processing CreateTablet for tablet 8c1491faa1be485792b6193d4ca22f37 (DEFAULT_TABLE table=default.Table_2e20136b218e40b6a54ab31e306e147e [id=ca323f84b00048e69461d15b7bd6369a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.461572   805 tablet_service.cc:1468] Processing CreateTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (DEFAULT_TABLE table=default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.462416   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5e1b0dbbc21b445d8115d963f6aba2a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.473663   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8c1491faa1be485792b6193d4ca22f37. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:14.425645   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 123175us
I20250628 07:52:14.478439  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5e1b0dbbc21b445d8115d963f6aba2a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.481431  1339 raft_consensus.cc:695] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:14.482273  1339 consensus_queue.cc:237] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.485172  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 950fea8f6bfb409f8868fe5e88d3dddb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.491273  1234 tablet_bootstrap.cc:492] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.491648  1234 ts_tablet_manager.cc:1397] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.196s	user 0.013s	sys 0.016s
I20250628 07:52:14.490990  1342 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:14.497771  1229 tablet_bootstrap.cc:654] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.500088  1292 raft_consensus.cc:491] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.500527  1292 raft_consensus.cc:513] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.502226  1292 leader_election.cc:290] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:14.504463  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:14.513319  1234 raft_consensus.cc:357] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.514050  1234 raft_consensus.cc:383] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.514353  1234 raft_consensus.cc:738] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.515038  1234 consensus_queue.cc:260] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.517421  1234 ts_tablet_manager.cc:1428] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.026s	user 0.006s	sys 0.001s
I20250628 07:52:14.517760  1230 tablet_bootstrap.cc:654] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.519685  1234 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.523772  1246 ts_tablet_manager.cc:1918] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:14.524148  1246 log.cc:1199] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:14.524830  1246 ts_tablet_manager.cc:1939] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:14.525328   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1c2ad1d6fc164caa9d9ca0dfa1229112"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:14.527004   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1c2ad1d6fc164caa9d9ca0dfa1229112 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.528123   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_929c4c35bfbb404cafec33d8710f7a1a" } modify_external_catalogs: true
I20250628 07:52:14.529687   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_b4f083a990fb4f3493ed3a44c9fb3fac [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 2 got 0
I20250628 07:52:14.533613   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_929c4c35bfbb404cafec33d8710f7a1a" } modify_external_catalogs: true
I20250628 07:52:14.535974  1234 tablet_bootstrap.cc:654] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.548535   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2952766630724aaaa5ed698e7c77fb35" } new_table_name: "default.Table_8244fa49d80542cca12bcda3b9225837" modify_external_catalogs: true
I20250628 07:52:14.558636   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:14.560024   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:14.560237   802 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:14.561441   802 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:14.549337  1352 tablet.cc:1722] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:14.571480  1234 tablet_bootstrap.cc:492] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.571866  1234 ts_tablet_manager.cc:1397] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.052s	user 0.009s	sys 0.002s
I20250628 07:52:14.574005  1234 raft_consensus.cc:357] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:14.575469   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet c84f2570b2da45d0b3ce6428723263a5 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c84f2570b2da45d0b3ce6428723263a5 already in progress: creating tablet
I20250628 07:52:14.576751  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.576956   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_2952766630724aaaa5ed698e7c77fb35 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 2 got 0
I20250628 07:52:14.577822  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:14.582986   823 raft_consensus.cc:3058] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.574599  1234 raft_consensus.cc:383] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.583379  1234 raft_consensus.cc:738] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.583801   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef449bd95fc64bf28c854b03a219c7b4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:14.583994  1234 consensus_queue.cc:260] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.591184  1234 ts_tablet_manager.cc:1428] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.019s	user 0.006s	sys 0.001s
I20250628 07:52:14.591565   963 tablet_service.cc:1468] Processing CreateTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb (DEFAULT_TABLE table=default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.592090  1234 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.592609   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 950fea8f6bfb409f8868fe5e88d3dddb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.596915  1234 tablet_bootstrap.cc:654] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.597769   965 tablet_service.cc:1468] Processing CreateTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 (DEFAULT_TABLE table=default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.598819   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_529002c755014ac185bde7c74488e8cf"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:14.600658   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_529002c755014ac185bde7c74488e8cf in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.604897   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4e9d585d5f0c4469828561a7df4fb917. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.612740   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6da92141ad354b5491ebafee908fe6d7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:14.614406   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6da92141ad354b5491ebafee908fe6d7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.625888  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:14.626574  1129 raft_consensus.cc:2466] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:14.627738   921 leader_election.cc:304] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:14.629006  1339 raft_consensus.cc:2802] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:14.629559  1339 raft_consensus.cc:491] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:14.629938  1339 raft_consensus.cc:3058] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.633877   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (table default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]) successfully deleted
I20250628 07:52:14.642144   964 tablet_service.cc:1468] Processing CreateTablet for tablet 8c1491faa1be485792b6193d4ca22f37 (DEFAULT_TABLE table=default.Table_2e20136b218e40b6a54ab31e306e147e [id=ca323f84b00048e69461d15b7bd6369a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.642992  1230 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.643194   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8c1491faa1be485792b6193d4ca22f37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.643390  1230 ts_tablet_manager.cc:1397] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.223s	user 0.010s	sys 0.000s
I20250628 07:52:14.646549   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8c61fc2b4d7d4c488a45ecb9a9121699"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:14.647993   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8c61fc2b4d7d4c488a45ecb9a9121699 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.649899   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_de1b1ac5b4e44da284d4cf266fef4876" } new_table_name: "default.Table_6d90b18a9f474219933cb8e8511d01d6" modify_external_catalogs: true
I20250628 07:52:14.666494  1230 raft_consensus.cc:357] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.666930   962 tablet_service.cc:1468] Processing CreateTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (DEFAULT_TABLE table=default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:14.667210  1230 raft_consensus.cc:383] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.667511  1230 raft_consensus.cc:738] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.668006   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5e1b0dbbc21b445d8115d963f6aba2a1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:14.668215  1230 consensus_queue.cc:260] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.674448  1230 ts_tablet_manager.cc:1428] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.031s	user 0.003s	sys 0.002s
I20250628 07:52:14.675755  1230 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:14.677114   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_a0e49295ed0e482eaa16a7761b01b32c [id=5e0bf38f7dc94bf59a2f39456ecae8fc], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 872ebf0393fa40cf82feebaad1ca3fa3 consensus state has no leader
I20250628 07:52:14.697897   966 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:14.698191   961 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:14.699573   757 leader_election.cc:304] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.702561  1230 tablet_bootstrap.cc:654] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:14.704689   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d39d14572c4647c7b4880c71a8707c08 already in progress: creating tablet
W20250628 07:52:14.705719   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fa96605dbabe4f0fbc26b057232960a4 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa96605dbabe4f0fbc26b057232960a4 already in progress: creating tablet
I20250628 07:52:14.705883  1246 tablet_replica.cc:331] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:14.706537  1246 raft_consensus.cc:2241] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:14.706614   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 2b51d6076e344c8f8e388b77b4b23928 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2b51d6076e344c8f8e388b77b4b23928 already in progress: creating tablet
I20250628 07:52:14.706957  1246 raft_consensus.cc:2270] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:14.708611  1246 ts_tablet_manager.cc:1905] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:14.718564  1229 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:14.718954  1229 ts_tablet_manager.cc:1397] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.508s	user 0.011s	sys 0.004s
I20250628 07:52:14.715615  1361 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:14.729266   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_2952766630724aaaa5ed698e7c77fb35 [id=4a58d7983dd146ae9e4ae45df899b0b6] alter complete (version 2)
I20250628 07:52:14.743467  1341 tablet.cc:1717] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Skipping requested alter to schema version 2, tablet already version 2
I20250628 07:52:14.721007  1229 raft_consensus.cc:357] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.749693  1229 raft_consensus.cc:383] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.749956  1229 raft_consensus.cc:738] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.754916  1339 raft_consensus.cc:513] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.757814   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:14.757740  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:14.758832  1129 raft_consensus.cc:3058] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:14.758970   920 leader_election.cc:343] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.762284  1292 raft_consensus.cc:2747] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.768201   539 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_218e7d52d78541a0b85a22af790324a6" } modify_external_catalogs: true
I20250628 07:52:14.768810   539 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_218e7d52d78541a0b85a22af790324a6" } modify_external_catalogs: true
I20250628 07:52:14.770475  1129 raft_consensus.cc:2466] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:14.772248   921 leader_election.cc:304] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:14.773124  1330 raft_consensus.cc:2802] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:14.773016   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_374f5d5ed8ee4ad481b7071e0d2865e1" } modify_external_catalogs: true
I20250628 07:52:14.773547  1330 raft_consensus.cc:695] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:14.773576   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_374f5d5ed8ee4ad481b7071e0d2865e1" } modify_external_catalogs: true
I20250628 07:52:14.774305  1330 consensus_queue.cc:237] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.772723  1362 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:14.780647  1230 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.781044  1230 ts_tablet_manager.cc:1397] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.105s	user 0.014s	sys 0.001s
I20250628 07:52:14.781641  1292 raft_consensus.cc:491] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.782104  1292 raft_consensus.cc:513] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } }
I20250628 07:52:14.784688  1292 leader_election.cc:290] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:14.750500  1229 consensus_queue.cc:260] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:14.786684   758 leader_election.cc:343] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: b8b1e09d0f224c10ab8df26f12239254
I20250628 07:52:14.790870  1230 raft_consensus.cc:357] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.791484  1230 raft_consensus.cc:383] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.791774  1230 raft_consensus.cc:738] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.792346  1230 consensus_queue.cc:260] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.794863  1229 ts_tablet_manager.cc:1428] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.076s	user 0.006s	sys 0.001s
I20250628 07:52:14.795624  1244 ts_tablet_manager.cc:1918] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:14.797791  1244 log.cc:1199] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c1a06bbe216f419598a587056f9b8d05
I20250628 07:52:14.800024  1230 ts_tablet_manager.cc:1428] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.019s	user 0.003s	sys 0.001s
I20250628 07:52:14.800918  1230 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:14.804491  1244 ts_tablet_manager.cc:1939] T c1a06bbe216f419598a587056f9b8d05 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:14.807096   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet c1a06bbe216f419598a587056f9b8d05 (table default.Table_8807b0cad1f9413aaa097f7c6158643a [id=b743e2df9053491ea56411f4059f190a]) successfully deleted
W20250628 07:52:14.808302   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2b51d6076e344c8f8e388b77b4b23928 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2b51d6076e344c8f8e388b77b4b23928 already in progress: creating tablet
W20250628 07:52:14.816573   757 leader_election.cc:343] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: b8b1e09d0f224c10ab8df26f12239254
I20250628 07:52:14.816812  1230 tablet_bootstrap.cc:654] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.816963   757 leader_election.cc:304] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.817689  1292 raft_consensus.cc:2747] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.821801  1229 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:14.826161  1339 leader_election.cc:290] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.827380   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_2952766630724aaaa5ed698e7c77fb35 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 2 got 0
I20250628 07:52:14.828349   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]). Expected version 3 got 0
I20250628 07:52:14.833770  1229 tablet_bootstrap.cc:654] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.835935  1230 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.837247  1230 ts_tablet_manager.cc:1397] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.037s	user 0.010s	sys 0.000s
I20250628 07:52:14.845052  1280 raft_consensus.cc:491] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.848425  1280 raft_consensus.cc:513] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.851207  1280 leader_election.cc:290] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:14.853614   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:14.854238   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet c84f2570b2da45d0b3ce6428723263a5 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c84f2570b2da45d0b3ce6428723263a5 already in progress: creating tablet
W20250628 07:52:14.854818   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d39d14572c4647c7b4880c71a8707c08 already in progress: creating tablet
I20250628 07:52:14.797879  1342 tablet.cc:1717] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Skipping requested alter to schema version 2, tablet already version 2
W20250628 07:52:14.855322   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fa96605dbabe4f0fbc26b057232960a4 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa96605dbabe4f0fbc26b057232960a4 already in progress: creating tablet
I20250628 07:52:14.857955  1361 tablet.cc:1717] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Skipping requested alter to schema version 2, tablet already version 2
I20250628 07:52:14.845353  1246 ts_tablet_manager.cc:1918] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:14.845364  1230 raft_consensus.cc:357] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:14.861932  1062 leader_election.cc:343] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.862198  1246 log.cc:1199] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:14.862309  1230 raft_consensus.cc:383] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.862645  1230 raft_consensus.cc:738] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.863368   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:14.863263  1230 consensus_queue.cc:260] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:14.864490  1062 leader_election.cc:343] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.864864  1062 leader_election.cc:304] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.865583  1230 ts_tablet_manager.cc:1428] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.024s	user 0.005s	sys 0.000s
I20250628 07:52:14.866225  1235 raft_consensus.cc:2747] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.866611  1230 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:14.872231  1230 tablet_bootstrap.cc:654] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.875159  1358 mvcc.cc:204] Tried to move back new op lower bound from 7172493862646939648 to 7172493858253451264. Current Snapshot: MvccSnapshot[applied={T|T < 7172493862646939648}]
I20250628 07:52:14.877715  1361 tablet.cc:1717] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Skipping requested alter to schema version 2, tablet already version 2
I20250628 07:52:14.886705  1234 tablet_bootstrap.cc:492] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.887202  1234 ts_tablet_manager.cc:1397] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.295s	user 0.029s	sys 0.052s
I20250628 07:52:14.889356  1367 tablet.cc:1722] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:14.891244  1246 ts_tablet_manager.cc:1939] T 0ac2a2d045864749841a9b5de03d140c P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:14.891456  1234 raft_consensus.cc:357] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.892071  1234 raft_consensus.cc:383] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.892319  1234 raft_consensus.cc:738] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.892896  1234 consensus_queue.cc:260] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.893769   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 0ac2a2d045864749841a9b5de03d140c (table default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]) successfully deleted
I20250628 07:52:14.894997  1234 ts_tablet_manager.cc:1428] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.007s	user 0.004s	sys 0.000s
W20250628 07:52:14.895323   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c84f2570b2da45d0b3ce6428723263a5 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c84f2570b2da45d0b3ce6428723263a5 already in progress: creating tablet
W20250628 07:52:14.896867   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fa96605dbabe4f0fbc26b057232960a4 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa96605dbabe4f0fbc26b057232960a4 already in progress: creating tablet
I20250628 07:52:14.897446  1230 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.897822  1230 ts_tablet_manager.cc:1397] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.031s	user 0.009s	sys 0.000s
W20250628 07:52:14.898772   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d39d14572c4647c7b4880c71a8707c08 already in progress: creating tablet
I20250628 07:52:14.900024  1230 raft_consensus.cc:357] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.918443  1230 raft_consensus.cc:383] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.918685  1230 raft_consensus.cc:738] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.923327  1230 consensus_queue.cc:260] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.925253  1230 ts_tablet_manager.cc:1428] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.027s	user 0.006s	sys 0.000s
I20250628 07:52:14.926048  1230 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:14.936542  1230 tablet_bootstrap.cc:654] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.939445  1360 raft_consensus.cc:491] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.939848  1360 raft_consensus.cc:513] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.941937  1360 leader_election.cc:290] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:14.943710   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:14.944654  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3637ab35a2b74ce3a111c77cbec8b0b0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:14.945396  1129 raft_consensus.cc:2391] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:14.946431   758 leader_election.cc:304] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:14.947377  1360 raft_consensus.cc:3058] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:14.951759  1234 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.957732  1234 tablet_bootstrap.cc:654] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.961663  1229 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:14.962234  1229 ts_tablet_manager.cc:1397] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.141s	user 0.020s	sys 0.013s
I20250628 07:52:14.963547  1360 raft_consensus.cc:2747] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:14.964841  1229 raft_consensus.cc:357] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.966336   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c62a447971794f91bd3b8d3b7957badf"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:14.966761  1229 raft_consensus.cc:383] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.966988  1229 raft_consensus.cc:738] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
W20250628 07:52:14.967809   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c62a447971794f91bd3b8d3b7957badf in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:14.967582  1229 consensus_queue.cc:260] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.975661  1234 tablet_bootstrap.cc:492] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:14.975921  1229 ts_tablet_manager.cc:1428] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20250628 07:52:14.976019  1234 ts_tablet_manager.cc:1397] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.024s	user 0.007s	sys 0.004s
I20250628 07:52:14.976668  1229 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:14.978235  1234 raft_consensus.cc:357] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.978818  1234 raft_consensus.cc:383] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.979112  1234 raft_consensus.cc:738] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.981225  1230 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:14.981600  1230 ts_tablet_manager.cc:1397] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.056s	user 0.010s	sys 0.000s
I20250628 07:52:14.983578  1230 raft_consensus.cc:357] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.984145  1230 raft_consensus.cc:383] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:14.984382  1230 raft_consensus.cc:738] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:14.988646  1234 consensus_queue.cc:260] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.992084  1280 raft_consensus.cc:491] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:14.992359  1229 tablet_bootstrap.cc:654] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:14.992493  1280 raft_consensus.cc:513] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.993279  1234 ts_tablet_manager.cc:1428] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.017s	user 0.006s	sys 0.000s
I20250628 07:52:14.994032  1234 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:14.994093  1280 leader_election.cc:290] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:14.995085  1230 consensus_queue.cc:260] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:14.995508   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67a81eb4342740e5bc16ee7f5eacb489" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:14.996904  1230 ts_tablet_manager.cc:1428] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.015s	user 0.004s	sys 0.000s
W20250628 07:52:14.997107  1062 leader_election.cc:343] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:14.997709  1230 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:15.002504  1230 tablet_bootstrap.cc:654] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.004869   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67a81eb4342740e5bc16ee7f5eacb489" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:15.006227  1062 leader_election.cc:343] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.006626  1062 leader_election.cc:304] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.007678  1280 raft_consensus.cc:2747] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.018496  1234 tablet_bootstrap.cc:654] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:15.021351   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 consensus state has no leader
I20250628 07:52:15.025151  1229 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:15.025544  1229 ts_tablet_manager.cc:1397] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.049s	user 0.006s	sys 0.005s
I20250628 07:52:15.027915  1230 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:15.031397  1229 raft_consensus.cc:357] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.032020  1229 raft_consensus.cc:383] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.032276  1229 raft_consensus.cc:738] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.037427  1229 consensus_queue.cc:260] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.044912  1229 ts_tablet_manager.cc:1428] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.019s	user 0.003s	sys 0.002s
I20250628 07:52:15.045727  1229 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:15.047922   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135] alter complete (version 1)
I20250628 07:52:15.050359   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 5e1b0dbbc21b445d8115d963f6aba2a1
I20250628 07:52:15.059861   538 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet a28a6de5789b41c4b038a543aa8f8cd8
I20250628 07:52:15.064370  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.071485   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.072154   805 tablet_service.cc:1515] Processing DeleteTablet for tablet b8b1e09d0f224c10ab8df26f12239254 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.074066  1229 tablet_bootstrap.cc:654] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.075258  1230 ts_tablet_manager.cc:1397] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.078s	user 0.010s	sys 0.000s
I20250628 07:52:15.077478  1244 tablet_replica.cc:331] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:15.077471  1230 raft_consensus.cc:357] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.078068  1230 raft_consensus.cc:383] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.078100  1244 raft_consensus.cc:2241] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:15.078312  1230 raft_consensus.cc:738] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.078472  1244 raft_consensus.cc:2270] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:15.079869  1244 ts_tablet_manager.cc:1905] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:15.079905  1234 tablet_bootstrap.cc:492] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:15.080309  1234 ts_tablet_manager.cc:1397] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.086s	user 0.007s	sys 0.002s
I20250628 07:52:15.082381  1234 raft_consensus.cc:357] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.082978  1234 raft_consensus.cc:383] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.083202  1234 raft_consensus.cc:738] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:15.083926   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5e1b0dbbc21b445d8115d963f6aba2a1 already in progress: creating tablet
I20250628 07:52:15.083714  1234 consensus_queue.cc:260] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.105316  1230 consensus_queue.cc:260] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:15.105871   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5e1b0dbbc21b445d8115d963f6aba2a1 already in progress: creating tablet
I20250628 07:52:15.106624  1230 ts_tablet_manager.cc:1428] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.031s	user 0.003s	sys 0.000s
I20250628 07:52:15.107389  1230 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:15.112298  1230 tablet_bootstrap.cc:654] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.088505  1234 ts_tablet_manager.cc:1428] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.001s
I20250628 07:52:15.114228  1234 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:15.119731  1234 tablet_bootstrap.cc:654] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.122217   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:37209
--webserver_interface=127.0.107.190
--webserver_port=42501
--builtin_ntp_servers=127.0.107.148:33331
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:52:15.131368   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f892200f501c448a90964969f585547c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.132944   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f892200f501c448a90964969f585547c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.137470   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4a45d5758c0f40209ba04b7d96752a10"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:15.147608  1309 raft_consensus.cc:491] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.148061  1309 raft_consensus.cc:513] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.149989  1309 leader_election.cc:290] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.150518  1309 raft_consensus.cc:491] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.150943  1309 raft_consensus.cc:513] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.152226   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:15.152616  1309 leader_election.cc:290] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.154379   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.155390   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.156844   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:15.157792  1062 leader_election.cc:343] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.158144  1360 raft_consensus.cc:491] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.158543  1360 raft_consensus.cc:513] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:15.159209  1062 leader_election.cc:343] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:15.159701  1062 leader_election.cc:343] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.160095  1360 leader_election.cc:290] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.160025  1062 leader_election.cc:304] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:15.160919  1062 leader_election.cc:343] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.161257  1062 leader_election.cc:304] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.161870  1235 raft_consensus.cc:2747] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.162356  1235 raft_consensus.cc:2747] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.163712   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.163736  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f7e2b290c69467aa42ca6328307bc3c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:15.164373  1129 raft_consensus.cc:2391] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:15.165359   758 leader_election.cc:304] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.166070  1360 raft_consensus.cc:3058] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.173465  1360 raft_consensus.cc:2747] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:15.158962   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4a45d5758c0f40209ba04b7d96752a10 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.193497   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_35424882587f46d1be44e2d8984956b7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.195084   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_35424882587f46d1be44e2d8984956b7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.203409   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:15.205329  1339 raft_consensus.cc:491] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.205766  1339 raft_consensus.cc:513] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.207530  1339 leader_election.cc:290] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.208917   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0ac2a2d045864749841a9b5de03d140c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:15.209517   824 raft_consensus.cc:2466] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:15.210151   921 leader_election.cc:343] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:15.210999   920 leader_election.cc:304] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:15.212183  1339 raft_consensus.cc:2802] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:15.212492  1339 raft_consensus.cc:491] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:15.212779  1339 raft_consensus.cc:3058] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.220345  1339 raft_consensus.cc:513] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:15.225636   921 leader_election.cc:343] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:15.225775   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0ac2a2d045864749841a9b5de03d140c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:15.226214   824 raft_consensus.cc:3058] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.227083   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_529002c755014ac185bde7c74488e8cf" } new_table_name: "default.Table_9e380ec7f38b4bda9c4a51ad3ba57b77" modify_external_catalogs: true
I20250628 07:52:15.229432  1360 raft_consensus.cc:491] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.229977  1360 raft_consensus.cc:513] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.231540  1339 leader_election.cc:290] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.232252  1360 leader_election.cc:290] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.233443  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:15.233989  1129 raft_consensus.cc:2466] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:15.234982   758 leader_election.cc:304] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:15.235663  1360 raft_consensus.cc:2802] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:15.235950  1360 raft_consensus.cc:491] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:15.236232  1360 raft_consensus.cc:3058] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.236349   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.236877   985 raft_consensus.cc:2466] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:15.242873  1360 raft_consensus.cc:513] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.235656   824 raft_consensus.cc:2466] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:15.244282   920 leader_election.cc:304] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:15.246994   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:15.247298  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "84a8065c2d1349b792308ac3aec1a8fe" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:15.247521   985 raft_consensus.cc:3058] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.247773  1129 raft_consensus.cc:3058] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.248615  1360 leader_election.cc:290] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.249276  1339 raft_consensus.cc:2802] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:15.250811  1339 raft_consensus.cc:695] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:15.254379  1339 consensus_queue.cc:237] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.265003   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6d90b18a9f474219933cb8e8511d01d6" } modify_external_catalogs: true
I20250628 07:52:15.265496   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6d90b18a9f474219933cb8e8511d01d6" } modify_external_catalogs: true
I20250628 07:52:15.267019   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b4f083a990fb4f3493ed3a44c9fb3fac" } new_table_name: "default.Table_55f07e6cb519419fa92ea6aeab0cad74" modify_external_catalogs: true
I20250628 07:52:15.303375  1129 raft_consensus.cc:2466] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:15.312542  1309 raft_consensus.cc:491] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.313006  1309 raft_consensus.cc:513] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.315187   985 raft_consensus.cc:2466] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:15.315721  1309 leader_election.cc:290] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.316344   757 leader_election.cc:304] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:15.317381  1360 raft_consensus.cc:2802] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:15.317885  1360 raft_consensus.cc:695] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:15.318842  1360 consensus_queue.cc:237] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.322130   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2df135033ee546f8bb9f4c4b54755b06" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:15.323450  1062 leader_election.cc:343] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.330875   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2df135033ee546f8bb9f4c4b54755b06" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:15.332077  1062 leader_election.cc:343] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.332438  1062 leader_election.cc:304] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.333401  1309 raft_consensus.cc:2747] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.339480  1339 consensus_queue.cc:1035] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:15.345713   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5e1b0dbbc21b445d8115d963f6aba2a1 already in progress: creating tablet
I20250628 07:52:15.349180  1292 raft_consensus.cc:491] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.349560  1292 raft_consensus.cc:513] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.351413  1292 leader_election.cc:290] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.353300   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.370194  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4a5d0deb0a049a2b6cc2ac5a8dd3c17" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:15.371003  1127 raft_consensus.cc:2391] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:15.374645   758 leader_election.cc:304] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.375900  1292 raft_consensus.cc:3058] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.390645  1230 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:15.391052  1230 ts_tablet_manager.cc:1397] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.284s	user 0.033s	sys 0.029s
I20250628 07:52:15.388880   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2d6aec0ecd124912b4fdcea46bff2243" } modify_external_catalogs: true
I20250628 07:52:15.392750   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2d6aec0ecd124912b4fdcea46bff2243" } modify_external_catalogs: true
I20250628 07:52:15.393157  1230 raft_consensus.cc:357] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.393728  1230 raft_consensus.cc:383] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.393950  1230 raft_consensus.cc:738] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.394608  1230 consensus_queue.cc:260] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.396780  1230 ts_tablet_manager.cc:1428] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:15.403205  1244 ts_tablet_manager.cc:1918] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:15.413024  1244 log.cc:1199] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/b8b1e09d0f224c10ab8df26f12239254
I20250628 07:52:15.414136  1292 raft_consensus.cc:2747] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.415400  1244 ts_tablet_manager.cc:1939] T b8b1e09d0f224c10ab8df26f12239254 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:15.421618  1230 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:15.426537  1230 tablet_bootstrap.cc:654] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.437857  1392 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:15.440593  1339 consensus_queue.cc:1035] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:15.445878  1330 consensus_queue.cc:1035] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:15.451550  1229 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:15.422525  1234 tablet_bootstrap.cc:492] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:15.444116  1367 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:15.452142  1234 ts_tablet_manager.cc:1397] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.338s	user 0.073s	sys 0.140s
I20250628 07:52:15.454209  1234 raft_consensus.cc:357] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.454746  1234 raft_consensus.cc:383] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.454986  1234 raft_consensus.cc:738] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.455560  1234 consensus_queue.cc:260] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.456032  1229 ts_tablet_manager.cc:1397] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.410s	user 0.019s	sys 0.032s
I20250628 07:52:15.457477  1394 tablet.cc:1722] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:15.462648  1230 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:15.463073  1230 ts_tablet_manager.cc:1397] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.042s	user 0.007s	sys 0.001s
I20250628 07:52:15.465346  1230 raft_consensus.cc:357] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.466737   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet b8b1e09d0f224c10ab8df26f12239254 (table default.Table_448f6b4c3c354e76aaf65492579adba1 [id=1340c5089505415585261910cb584f6e]) successfully deleted
I20250628 07:52:15.467837  1230 raft_consensus.cc:383] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.468266  1230 raft_consensus.cc:738] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.471227  1235 raft_consensus.cc:491] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.477603  1230 consensus_queue.cc:260] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.477888  1235 raft_consensus.cc:513] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.479589  1230 ts_tablet_manager.cc:1428] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.016s	user 0.003s	sys 0.000s
I20250628 07:52:15.479715  1235 leader_election.cc:290] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.480427  1230 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:15.481688   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:15.483057  1062 leader_election.cc:343] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.484525   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:15.486212  1062 leader_election.cc:343] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.486601  1062 leader_election.cc:304] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.487651  1309 raft_consensus.cc:491] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.488083  1309 raft_consensus.cc:513] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.489985  1309 leader_election.cc:290] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.490396  1309 raft_consensus.cc:2747] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.492645   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.495393   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:15.496645  1062 leader_election.cc:343] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:15.497357  1062 leader_election.cc:343] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.497776  1062 leader_election.cc:304] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.498817  1235 raft_consensus.cc:2747] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.498922  1234 ts_tablet_manager.cc:1428] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.047s	user 0.011s	sys 0.000s
I20250628 07:52:15.500428  1234 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:15.458731  1229 raft_consensus.cc:357] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.513435  1229 raft_consensus.cc:383] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.518812  1229 raft_consensus.cc:738] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.521848  1230 tablet_bootstrap.cc:654] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:15.527740   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 4fedd3171e2944b1a9c8fafcacafb2a1 has no consensus state
I20250628 07:52:15.528513   546 master_service.cc:948] ReplaceTablet: received request to replace tablet 2df135033ee546f8bb9f4c4b54755b06 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:15.529997   544 master_service.cc:948] ReplaceTablet: received request to replace tablet ef449bd95fc64bf28c854b03a219c7b4 from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:15.536863   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2d77be1f37d2400f842c01019e4eb474 consensus state has no leader
I20250628 07:52:15.540266  1229 consensus_queue.cc:260] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.554172  1229 ts_tablet_manager.cc:1428] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.098s	user 0.001s	sys 0.005s
I20250628 07:52:15.555003  1229 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:15.557920  1330 consensus_queue.cc:1035] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:15.575465  1234 tablet_bootstrap.cc:654] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.595819  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 (DEFAULT_TABLE table=default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.596771  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29acadb819ef46318e2ba5ba2d3c3c37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.603165   965 tablet_service.cc:1468] Processing CreateTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 (DEFAULT_TABLE table=default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.604120   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29acadb819ef46318e2ba5ba2d3c3c37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.605784   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d74975b9cd2942c98865802e679ef4d5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.607813   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d74975b9cd2942c98865802e679ef4d5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.616277   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:15.617391  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 482095693aac476faf997ea8e1a4b460 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.617846   544 catalog_manager.cc:6652] ReplaceTablet: tablet ef449bd95fc64bf28c854b03a219c7b4 deleted and replaced by tablet 8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:15.618444  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 482095693aac476faf997ea8e1a4b460. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.622321   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (table default.Table_2d6aec0ecd124912b4fdcea46bff2243 [id=853998210b39471f9d2d0bf0ebf072d5]). Expected version 1 got 0
I20250628 07:52:15.622977   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]). Expected version 1 got 0
I20250628 07:52:15.623538   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:15.626384   963 tablet_service.cc:1468] Processing CreateTablet for tablet 482095693aac476faf997ea8e1a4b460 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.627347   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 482095693aac476faf997ea8e1a4b460. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.629302   964 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:15.630052  1383 tablet_replica.cc:331] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:15.631369  1383 raft_consensus.cc:2241] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:15.631937  1383 raft_consensus.cc:2270] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:15.636135  1229 tablet_bootstrap.cc:654] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.640292  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.644394  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 71da5208ceef4186855cbe471f0fd8a3 (DEFAULT_TABLE table=default.Table_177b8783c82149aabfddcfe195d451a7 [id=3f993fcff5574851b92244167bb2ee8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.645330  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71da5208ceef4186855cbe471f0fd8a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.654067   962 tablet_service.cc:1468] Processing CreateTablet for tablet d62ecf32bcc54310b9114d5655be7673 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.655021   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d62ecf32bcc54310b9114d5655be7673. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.656778   964 tablet_service.cc:1468] Processing CreateTablet for tablet 71da5208ceef4186855cbe471f0fd8a3 (DEFAULT_TABLE table=default.Table_177b8783c82149aabfddcfe195d451a7 [id=3f993fcff5574851b92244167bb2ee8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.658897  1103 tablet_service.cc:1468] Processing CreateTablet for tablet d62ecf32bcc54310b9114d5655be7673 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.659669   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71da5208ceef4186855cbe471f0fd8a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.659931  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d62ecf32bcc54310b9114d5655be7673. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.670782   805 tablet_service.cc:1468] Processing CreateTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 (DEFAULT_TABLE table=default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.673271   803 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.673273  1235 raft_consensus.cc:491] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.674350   802 tablet_service.cc:1468] Processing CreateTablet for tablet 71da5208ceef4186855cbe471f0fd8a3 (DEFAULT_TABLE table=default.Table_177b8783c82149aabfddcfe195d451a7 [id=3f993fcff5574851b92244167bb2ee8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.673686  1235 raft_consensus.cc:513] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.675555   798 tablet_service.cc:1468] Processing CreateTablet for tablet d62ecf32bcc54310b9114d5655be7673 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.676561  1235 leader_election.cc:290] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:15.677315   801 tablet_service.cc:1468] Processing CreateTablet for tablet 482095693aac476faf997ea8e1a4b460 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.678330  1234 tablet_bootstrap.cc:492] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:15.678741  1234 ts_tablet_manager.cc:1397] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.178s	user 0.011s	sys 0.044s
I20250628 07:52:15.680022   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa96605dbabe4f0fbc26b057232960a4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:15.681267  1062 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.682591   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa96605dbabe4f0fbc26b057232960a4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:15.686769  1062 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:15.687166  1062 leader_election.cc:304] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:15.688937   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d62ecf32bcc54310b9114d5655be7673. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.690016  1309 raft_consensus.cc:2747] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:15.690661   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:15.698120  1383 ts_tablet_manager.cc:1905] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:15.697799  1234 raft_consensus.cc:357] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.698457  1234 raft_consensus.cc:383] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.695511   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:15.698722  1234 raft_consensus.cc:738] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.699261  1234 consensus_queue.cc:260] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:15.696084   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (table default.Table_6d90b18a9f474219933cb8e8511d01d6 [id=4d5813f4497543018828699e64223907], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 consensus state has no leader
I20250628 07:52:15.700217   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.701905  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.703524   546 catalog_manager.cc:6652] ReplaceTablet: tablet 2df135033ee546f8bb9f4c4b54755b06 deleted and replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:15.703601   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.704619  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.701471   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:15.705204   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:15.707446   803 tablet_service.cc:1468] Processing CreateTablet for tablet 16a8bee148664e948529d32c0cf2c793 (DEFAULT_TABLE table=default.Table_54b2cdede05f442cb652727e32c46009 [id=f59d42b137cf4536a002585b728017f6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.708714  1244 tablet_replica.cc:331] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:15.709441  1244 raft_consensus.cc:2241] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:15.709889  1244 raft_consensus.cc:2270] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:15.709734   966 tablet_service.cc:1468] Processing CreateTablet for tablet 16a8bee148664e948529d32c0cf2c793 (DEFAULT_TABLE table=default.Table_54b2cdede05f442cb652727e32c46009 [id=f59d42b137cf4536a002585b728017f6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:15.710602   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 16a8bee148664e948529d32c0cf2c793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.710676  1234 ts_tablet_manager.cc:1428] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.001s	sys 0.006s
I20250628 07:52:15.711475  1234 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:15.711648   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 482095693aac476faf997ea8e1a4b460. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.718554  1244 ts_tablet_manager.cc:1905] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:15.721560   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29acadb819ef46318e2ba5ba2d3c3c37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.723460  1397 raft_consensus.cc:491] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.723870  1397 raft_consensus.cc:513] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.725704   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71da5208ceef4186855cbe471f0fd8a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.726226  1397 leader_election.cc:290] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:15.728732   758 leader_election.cc:343] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:15.731222   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 16a8bee148664e948529d32c0cf2c793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.733544   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e71a4b8cfa3a41d1837ca6fd2e2a3e67" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.734182   985 raft_consensus.cc:2466] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:15.741313   757 leader_election.cc:304] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:15.742844  1397 raft_consensus.cc:2802] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:15.743160  1397 raft_consensus.cc:491] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:15.744071  1397 raft_consensus.cc:3058] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.744093   539 catalog_manager.cc:5582] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:15.753036   539 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:15.753819   539 catalog_manager.cc:5582] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:15.755398  1397 raft_consensus.cc:513] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.760473   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e71a4b8cfa3a41d1837ca6fd2e2a3e67" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:15.760977   985 raft_consensus.cc:3058] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.763777  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 16a8bee148664e948529d32c0cf2c793 (DEFAULT_TABLE table=default.Table_54b2cdede05f442cb652727e32c46009 [id=f59d42b137cf4536a002585b728017f6]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:15.763999   758 leader_election.cc:343] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:15.764418  1397 leader_election.cc:290] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.764799  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 16a8bee148664e948529d32c0cf2c793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:15.770037  1234 tablet_bootstrap.cc:654] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.779188  1234 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:15.779546  1234 ts_tablet_manager.cc:1397] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.068s	user 0.006s	sys 0.002s
I20250628 07:52:15.780143  1330 consensus_queue.cc:1035] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:15.781556  1234 raft_consensus.cc:357] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.782063  1234 raft_consensus.cc:383] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.782269  1234 raft_consensus.cc:738] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.782836  1234 consensus_queue.cc:260] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.788451  1234 ts_tablet_manager.cc:1428] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.009s	user 0.005s	sys 0.001s
I20250628 07:52:15.790886  1234 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:15.796949  1234 tablet_bootstrap.cc:654] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:15.818212   531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20250628 07:52:15.820876   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20250628 07:52:15.755635   539 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]). Expected version 1 got 0
I20250628 07:52:15.828480  1234 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:15.828872  1234 ts_tablet_manager.cc:1397] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.040s	user 0.004s	sys 0.004s
I20250628 07:52:15.831136  1234 raft_consensus.cc:357] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.831736  1234 raft_consensus.cc:383] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:15.832002  1234 raft_consensus.cc:738] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:15.837548   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 3637ab35a2b74ce3a111c77cbec8b0b0
W20250628 07:52:15.839851   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 32) took 755 ms (client timeout 1000 ms). Trace:
I20250628 07:52:15.841943  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.842082   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:15.842826   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:15.840207   538 rpcz_store.cc:269] 0628 07:52:15.084784 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.087434 (+  2650us) service_pool.cc:225] Handling call
0628 07:52:15.839789 (+752355us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":7}
I20250628 07:52:15.832597  1234 consensus_queue.cc:260] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.846325  1234 ts_tablet_manager.cc:1428] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.017s	user 0.004s	sys 0.000s
I20250628 07:52:15.847170  1234 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:15.870766  1234 tablet_bootstrap.cc:654] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:15.880156  1330 raft_consensus.cc:491] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.880676  1330 raft_consensus.cc:513] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.883471   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:15.884156   823 raft_consensus.cc:2466] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:15.885334   920 leader_election.cc:304] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:15.886838  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.887847  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:15.888401  1129 raft_consensus.cc:2466] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:15.890017  1330 leader_election.cc:290] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.891010  1330 raft_consensus.cc:2802] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:15.891742  1330 raft_consensus.cc:491] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:15.892179  1330 raft_consensus.cc:3058] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.914373  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.916451   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_92381327fd4e479298ecf33ccdd4eccb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.918372   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_92381327fd4e479298ecf33ccdd4eccb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.919405   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9e380ec7f38b4bda9c4a51ad3ba57b77" } modify_external_catalogs: true
I20250628 07:52:15.919952   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9e380ec7f38b4bda9c4a51ad3ba57b77" } modify_external_catalogs: true
I20250628 07:52:15.922670   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b2f11d5c89e84c33a307bcc83159841c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.924130   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b2f11d5c89e84c33a307bcc83159841c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.929440   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ed752c51c98743c6b11f0f3a2ce3db69" } new_table_name: "default.Table_0b57fdd311634de995ecfba2548d0468" modify_external_catalogs: true
I20250628 07:52:15.938652   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1c2ad1d6fc164caa9d9ca0dfa1229112" } new_table_name: "default.Table_5e720c1e9255472196788707be8302ec" modify_external_catalogs: true
I20250628 07:52:15.942032   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a148c40376f64b4fb79113972fe5693f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:15.943712   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a148c40376f64b4fb79113972fe5693f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:15.947521   985 raft_consensus.cc:2466] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:15.949415  1397 raft_consensus.cc:491] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:15.949926  1397 raft_consensus.cc:513] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:15.952090  1397 leader_election.cc:290] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:15.956497  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:15.958052  1129 raft_consensus.cc:2466] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:15.959172   758 leader_election.cc:304] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:15.960109  1397 raft_consensus.cc:2802] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:15.960510  1397 raft_consensus.cc:491] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:15.960880  1397 raft_consensus.cc:3058] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:15.977689  1397 raft_consensus.cc:513] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.000090  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:16.000234   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 33) took 797 ms (client timeout 1000 ms). Trace:
I20250628 07:52:16.001317  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:15.955485   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:15.977770  1408 consensus_queue.cc:1035] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:16.023918   757 leader_election.cc:343] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.026777   757 leader_election.cc:304] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:16.045068   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c096e0d11db417187a1bde167c0735f" } modify_external_catalogs: true
I20250628 07:52:16.045652   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c096e0d11db417187a1bde167c0735f" } modify_external_catalogs: true
I20250628 07:52:16.106887  1402 tablet_replica.cc:331] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.107928  1402 raft_consensus.cc:2241] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
W20250628 07:52:16.000538   543 rpcz_store.cc:269] 0628 07:52:15.202587 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.205872 (+  3285us) service_pool.cc:225] Handling call
0628 07:52:16.000177 (+794305us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":83600,"reported_tablets":9}
I20250628 07:52:16.135341  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:16.135854  1129 raft_consensus.cc:3058] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.144691  1129 raft_consensus.cc:2466] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:16.146009   758 leader_election.cc:304] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:16.151302  1408 consensus_queue.cc:1035] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:16.165066   534 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 142641us
I20250628 07:52:16.171435   542 master_service.cc:948] ReplaceTablet: received request to replace tablet ef449bd95fc64bf28c854b03a219c7b4 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:16.175745  1416 raft_consensus.cc:2802] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:16.184497  1415 raft_consensus.cc:2802] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:16.189409  1417 raft_consensus.cc:491] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.189805  1417 raft_consensus.cc:513] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.191588  1417 leader_election.cc:290] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:16.192734  1416 raft_consensus.cc:695] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:16.193490  1416 consensus_queue.cc:237] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:16.197220   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 170492us
I20250628 07:52:16.197871  1230 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.198323  1230 ts_tablet_manager.cc:1397] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.718s	user 0.010s	sys 0.004s
W20250628 07:52:16.207726   921 consensus_peers.cc:489] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:16.208475  1230 raft_consensus.cc:357] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.209136  1230 raft_consensus.cc:383] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.209371  1230 raft_consensus.cc:738] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.209957  1230 consensus_queue.cc:260] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.212005  1230 ts_tablet_manager.cc:1428] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:16.212806  1230 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.218060  1235 raft_consensus.cc:491] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.218464  1235 raft_consensus.cc:513] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.220340  1235 leader_election.cc:290] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.222733  1230 tablet_bootstrap.cc:654] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.225401   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2b51d6076e344c8f8e388b77b4b23928" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.226511  1062 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.227383   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:16.228163  1415 raft_consensus.cc:695] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:16.231243  1415 consensus_queue.cc:237] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:16.233204   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 87294.4us
I20250628 07:52:16.236160  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:16.236728  1126 raft_consensus.cc:2466] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:16.238099   758 leader_election.cc:304] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
W20250628 07:52:16.240084  1229 thread.cc:640] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.589s	user 0.050s	sys 0.087s
W20250628 07:52:16.240401  1229 thread.cc:606] log-alloc [worker] (thread pool) Time spent starting thread: real 0.590s	user 0.051s	sys 0.088s
I20250628 07:52:16.274758  1230 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.275162  1230 ts_tablet_manager.cc:1397] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.063s	user 0.008s	sys 0.000s
I20250628 07:52:16.274955   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:16.278026  1230 raft_consensus.cc:357] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.277562  1234 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:16.278640  1230 raft_consensus.cc:383] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.278882  1230 raft_consensus.cc:738] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.279037  1234 ts_tablet_manager.cc:1397] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.432s	user 0.040s	sys 0.066s
W20250628 07:52:16.290815   963 consensus_meta.cc:287] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.503s	user 0.006s	sys 0.001s
I20250628 07:52:16.301290  1230 consensus_queue.cc:260] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.303354  1230 ts_tablet_manager.cc:1428] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.028s	user 0.006s	sys 0.000s
I20250628 07:52:16.304167  1230 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:16.304416   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 66397us
I20250628 07:52:16.304831  1415 raft_consensus.cc:2802] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:16.305187  1415 raft_consensus.cc:491] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:16.305472  1415 raft_consensus.cc:3058] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.306114  1229 tablet_bootstrap.cc:492] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
W20250628 07:52:16.306306  1104 consensus_meta.cc:287] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.651s	user 0.007s	sys 0.000s
I20250628 07:52:16.306494  1229 ts_tablet_manager.cc:1397] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.752s	user 0.055s	sys 0.091s
I20250628 07:52:16.308482  1229 raft_consensus.cc:357] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.309065  1229 raft_consensus.cc:383] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.309320  1229 raft_consensus.cc:738] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.311087  1234 raft_consensus.cc:357] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.311553  1402 raft_consensus.cc:2270] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.311708  1234 raft_consensus.cc:383] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.311949  1234 raft_consensus.cc:738] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.312448  1234 consensus_queue.cc:260] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.313716  1415 raft_consensus.cc:513] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.315402  1415 leader_election.cc:290] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:16.317711  1105 thread.cc:640] tablet-delete [worker] (thread pool) Time spent creating pthread: real 0.677s	user 0.010s	sys 0.004s
I20250628 07:52:16.317836  1234 ts_tablet_manager.cc:1428] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.038s	user 0.005s	sys 0.000s
W20250628 07:52:16.318089  1105 thread.cc:606] tablet-delete [worker] (thread pool) Time spent starting thread: real 0.677s	user 0.010s	sys 0.004s
I20250628 07:52:16.318611  1234 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:16.322793   538 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 4fedd3171e2944b1a9c8fafcacafb2a1
I20250628 07:52:16.327147  1230 tablet_bootstrap.cc:654] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.309854  1229 consensus_queue.cc:260] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.330771  1339 raft_consensus.cc:491] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.331197  1339 raft_consensus.cc:513] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.334285  1229 ts_tablet_manager.cc:1428] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.028s	user 0.000s	sys 0.006s
I20250628 07:52:16.334721  1234 tablet_bootstrap.cc:654] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.335083  1229 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:16.339123  1339 leader_election.cc:290] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:16.340813  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:16.342458   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:16.342620  1128 raft_consensus.cc:2391] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
W20250628 07:52:16.343938   757 leader_election.cc:343] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.344074  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f22700fc70d4f14ae51a834900db22b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
W20250628 07:52:16.344460   757 leader_election.cc:343] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.344663  1127 raft_consensus.cc:3058] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.346230  1415 consensus_queue.cc:1035] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:16.349815  1416 consensus_queue.cc:1035] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:16.353284   757 leader_election.cc:343] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.356571   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2d77be1f37d2400f842c01019e4eb474" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:16.358938   920 leader_election.cc:304] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:16.359788  1339 raft_consensus.cc:3058] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.363289  1230 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.363682  1230 ts_tablet_manager.cc:1397] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.060s	user 0.008s	sys 0.000s
I20250628 07:52:16.369077  1230 raft_consensus.cc:357] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.369752  1230 raft_consensus.cc:383] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.370010  1230 raft_consensus.cc:738] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.373231  1339 raft_consensus.cc:2747] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.377113  1230 consensus_queue.cc:260] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.379825  1230 ts_tablet_manager.cc:1428] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.016s	user 0.007s	sys 0.000s
I20250628 07:52:16.382378  1230 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.390331  1402 ts_tablet_manager.cc:1905] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:16.392829   964 consensus_meta.cc:287] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.567s	user 0.004s	sys 0.002s
W20250628 07:52:16.397300   962 consensus_meta.cc:287] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.700s	user 0.005s	sys 0.006s
I20250628 07:52:16.397722  1230 tablet_bootstrap.cc:654] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.409258  1230 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.409623  1230 ts_tablet_manager.cc:1397] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.029s	user 0.011s	sys 0.000s
I20250628 07:52:16.411803  1230 raft_consensus.cc:357] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.412622  1230 raft_consensus.cc:383] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.412942  1230 raft_consensus.cc:738] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.417829  1230 consensus_queue.cc:260] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.420831  1230 ts_tablet_manager.cc:1428] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.000s
W20250628 07:52:16.430900  1106 consensus_meta.cc:287] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.572s	user 0.008s	sys 0.000s
I20250628 07:52:16.438366  1234 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:16.438793  1234 ts_tablet_manager.cc:1397] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.120s	user 0.011s	sys 0.051s
I20250628 07:52:16.441288  1234 raft_consensus.cc:357] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.441859  1234 raft_consensus.cc:383] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.442085  1234 raft_consensus.cc:738] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.442617  1234 consensus_queue.cc:260] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.204594  1244 ts_tablet_manager.cc:1918] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:16.198220  1397 leader_election.cc:290] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:16.221956   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2b51d6076e344c8f8e388b77b4b23928" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:16.447532  1062 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.448148  1244 log.cc:1199] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:16.447988  1062 leader_election.cc:304] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.448961  1235 raft_consensus.cc:2747] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.452219  1229 tablet_bootstrap.cc:654] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.452670  1234 ts_tablet_manager.cc:1428] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.014s	user 0.002s	sys 0.006s
I20250628 07:52:16.459856  1127 raft_consensus.cc:2466] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:16.461160  1244 ts_tablet_manager.cc:1939] T ef449bd95fc64bf28c854b03a219c7b4 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:16.461202   758 leader_election.cc:304] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.462213  1397 raft_consensus.cc:2802] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:16.462705  1230 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.464174  1397 raft_consensus.cc:695] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:16.465116  1397 consensus_queue.cc:237] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.469045  1230 tablet_bootstrap.cc:654] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.470414  1235 raft_consensus.cc:491] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.470824  1235 raft_consensus.cc:513] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:16.471474  1107 consensus_meta.cc:287] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.558s	user 0.010s	sys 0.001s
I20250628 07:52:16.472543  1235 leader_election.cc:290] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.474385   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7a3e80af50b542ce94ea3cb2b2cef3b8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:16.475602  1062 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.478466   541 master_service.cc:948] ReplaceTablet: received request to replace tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:16.502118  1234 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:16.502388  1402 ts_tablet_manager.cc:1918] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:16.506616  1234 tablet_bootstrap.cc:654] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.506990  1402 log.cc:1199] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:16.507872  1402 ts_tablet_manager.cc:1939] T ef449bd95fc64bf28c854b03a219c7b4 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:16.510982  1402 tablet_replica.cc:331] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.511140   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
I20250628 07:52:16.522424  1402 raft_consensus.cc:2241] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.523073  1402 raft_consensus.cc:2270] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.524682   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
W20250628 07:52:16.526857   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 already in progress: creating tablet
W20250628 07:52:16.528761   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2df135033ee546f8bb9f4c4b54755b06 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2df135033ee546f8bb9f4c4b54755b06 already in progress: creating tablet
I20250628 07:52:16.529857  1244 tablet_replica.cc:331] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:16.530689  1244 raft_consensus.cc:2241] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.538475  1402 ts_tablet_manager.cc:1905] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:16.541590   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d3f94717f2e3408abde6e223f1920e54"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:16.543392   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d3f94717f2e3408abde6e223f1920e54 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:16.559553   920 consensus_peers.cc:489] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:16.561769   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7a3e80af50b542ce94ea3cb2b2cef3b8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.567968  1062 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.570451  1062 leader_election.cc:304] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:16.571682  1062 raft_consensus.cc:2684] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: unable to run election callback: Service unavailable: Thread pool token was shut down
I20250628 07:52:16.579253  1235 raft_consensus.cc:491] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.579789  1235 raft_consensus.cc:513] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.581915  1235 leader_election.cc:290] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.585469   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.590736  1330 consensus_meta.cc:287] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.698s	user 0.009s	sys 0.001s
I20250628 07:52:16.591151  1330 raft_consensus.cc:513] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:16.594954  1062 leader_election.cc:343] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.598006  1383 ts_tablet_manager.cc:1918] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:16.601337  1383 log.cc:1199] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:16.602375  1383 ts_tablet_manager.cc:1939] T ef449bd95fc64bf28c854b03a219c7b4 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:16.603304  1330 leader_election.cc:290] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:16.607057  1402 ts_tablet_manager.cc:1918] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:16.607703  1402 log.cc:1199] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:16.608583  1402 ts_tablet_manager.cc:1939] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:16.583750   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:16.593951   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:16.609741   825 raft_consensus.cc:3058] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.611552   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (table default.Table_6d90b18a9f474219933cb8e8511d01d6 [id=4d5813f4497543018828699e64223907]) successfully deleted
I20250628 07:52:16.610839  1244 raft_consensus.cc:2270] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.615320  1244 ts_tablet_manager.cc:1905] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:16.616307   825 raft_consensus.cc:2466] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:16.617574  1062 leader_election.cc:343] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.618146  1062 leader_election.cc:304] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.618867  1235 raft_consensus.cc:2747] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.622565  1402 tablet_replica.cc:331] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.623462  1402 raft_consensus.cc:2241] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.623984  1383 tablet_replica.cc:331] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:16.624876  1402 raft_consensus.cc:2270] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.626176  1229 tablet_bootstrap.cc:492] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:16.627815  1383 raft_consensus.cc:2241] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:16.628533  1402 ts_tablet_manager.cc:1905] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:16.645792   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 60179e6765b145adbf637a8435eef724 has no consensus state
I20250628 07:52:16.647166   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c61fc2b4d7d4c488a45ecb9a9121699" } modify_external_catalogs: true
I20250628 07:52:16.647806   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c61fc2b4d7d4c488a45ecb9a9121699" } modify_external_catalogs: true
I20250628 07:52:16.664134  1446 raft_consensus.cc:491] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.664790  1446 raft_consensus.cc:513] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.668048   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:16.670274  1062 leader_election.cc:343] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.672071  1446 leader_election.cc:290] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.682622   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:16.710781  1230 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.711210  1230 ts_tablet_manager.cc:1397] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.249s	user 0.029s	sys 0.026s
I20250628 07:52:16.713280  1230 raft_consensus.cc:357] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.713848  1230 raft_consensus.cc:383] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.714082  1230 raft_consensus.cc:738] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.714601  1230 consensus_queue.cc:260] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.716495  1230 ts_tablet_manager.cc:1428] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:16.718134  1230 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.723444  1230 tablet_bootstrap.cc:654] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:16.725739   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3], current schema version=3) failed: Not found: failed to reset TS proxy: No leader found: Tablet 2d77be1f37d2400f842c01019e4eb474 consensus state has no leader
I20250628 07:52:16.725800  1402 ts_tablet_manager.cc:1918] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:16.726168  1402 log.cc:1199] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:16.726573  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:16.726831  1402 ts_tablet_manager.cc:1939] T 2df135033ee546f8bb9f4c4b54755b06 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:16.731066  1402 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 92) took 1026 ms. Trace:
I20250628 07:52:16.737166  1402 rpcz_store.cc:276] 0628 07:52:15.704305 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.704440 (+   135us) service_pool.cc:225] Handling call
0628 07:52:16.611060 (+906620us) ts_tablet_manager.cc:1145] Deleting tablet 2df135033ee546f8bb9f4c4b54755b06
0628 07:52:16.622443 (+ 11383us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:16.670411 (+ 47968us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:16.700972 (+ 30561us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:16.731005 (+ 30033us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":906074}
I20250628 07:52:16.738665  1402 tablet_replica.cc:331] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.739454   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 2df135033ee546f8bb9f4c4b54755b06 (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:16.742174   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7dc8fa39558f4514bbe045fbd9c99910"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:16.743764   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7dc8fa39558f4514bbe045fbd9c99910 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:16.745796  1402 raft_consensus.cc:2241] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.746333  1402 raft_consensus.cc:2270] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.746006   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5f59babfe3234a8e83bfab516b219032"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:16.747817   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5f59babfe3234a8e83bfab516b219032 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:16.748908  1402 ts_tablet_manager.cc:1905] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:16.751204   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 263597us
I20250628 07:52:16.759754  1230 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.760213  1230 ts_tablet_manager.cc:1397] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.042s	user 0.004s	sys 0.004s
I20250628 07:52:16.762871  1230 raft_consensus.cc:357] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.763597  1230 raft_consensus.cc:383] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.763875  1230 raft_consensus.cc:738] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.768129  1230 consensus_queue.cc:260] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.782850   920 leader_election.cc:304] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:16.783995  1230 ts_tablet_manager.cc:1428] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.023s	user 0.007s	sys 0.001s
I20250628 07:52:16.784850  1230 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.785581   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet ef449bd95fc64bf28c854b03a219c7b4 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
W20250628 07:52:16.786187   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 already in progress: creating tablet
W20250628 07:52:16.786715   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2df135033ee546f8bb9f4c4b54755b06 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2df135033ee546f8bb9f4c4b54755b06 already in progress: creating tablet
I20250628 07:52:16.786690  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:16.788889  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:16.791489   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.793649  1062 leader_election.cc:343] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.794068  1062 leader_election.cc:304] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.794467  1229 ts_tablet_manager.cc:1397] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.459s	user 0.008s	sys 0.004s
I20250628 07:52:16.795621  1230 tablet_bootstrap.cc:654] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.797230  1446 raft_consensus.cc:2747] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.798513  1229 raft_consensus.cc:357] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.799463  1229 raft_consensus.cc:383] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.799753  1229 raft_consensus.cc:738] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.800381  1229 consensus_queue.cc:260] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.802773  1229 ts_tablet_manager.cc:1428] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.003s	sys 0.002s
I20250628 07:52:16.808228  1229 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:16.811549  1402 ts_tablet_manager.cc:1918] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:16.829293  1402 log.cc:1199] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/3637ab35a2b74ce3a111c77cbec8b0b0
I20250628 07:52:16.830534  1402 ts_tablet_manager.cc:1939] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:16.834339  1230 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.834803  1230 ts_tablet_manager.cc:1397] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.050s	user 0.007s	sys 0.003s
I20250628 07:52:16.836143  1430 raft_consensus.cc:2802] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:16.836675  1430 raft_consensus.cc:695] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:16.837249  1230 raft_consensus.cc:357] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.838052  1230 raft_consensus.cc:383] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.837626  1430 consensus_queue.cc:237] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.838397  1230 raft_consensus.cc:738] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.839892   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (table default.Table_2d6aec0ecd124912b4fdcea46bff2243 [id=853998210b39471f9d2d0bf0ebf072d5]) successfully deleted
I20250628 07:52:16.840374  1230 consensus_queue.cc:260] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.841051  1446 raft_consensus.cc:491] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.842047  1446 raft_consensus.cc:513] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.843608   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet ffe3bcf321fb4521bcd0037f380e267c
I20250628 07:52:16.843685  1446 leader_election.cc:290] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.844636   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:16.846282  1062 leader_election.cc:343] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.847469   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.849458  1062 leader_election.cc:343] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.849813  1062 leader_election.cc:304] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.850479  1446 raft_consensus.cc:2747] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.839742  1402 tablet_replica.cc:331] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.854091  1402 raft_consensus.cc:2241] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.862291  1402 raft_consensus.cc:2270] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.862747  1234 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:16.861819  1230 ts_tablet_manager.cc:1428] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.027s	user 0.005s	sys 0.000s
I20250628 07:52:16.863168  1234 ts_tablet_manager.cc:1397] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.361s	user 0.031s	sys 0.074s
I20250628 07:52:16.866773  1402 ts_tablet_manager.cc:1905] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:16.867449  1229 tablet_bootstrap.cc:654] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.867272  1230 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.869835  1383 raft_consensus.cc:2270] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:16.870817   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 339374us
I20250628 07:52:16.873606  1383 ts_tablet_manager.cc:1905] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:16.874581  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "deb5a9ae7ab5463cae35ff1086fd8348" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
W20250628 07:52:16.877473   921 leader_election.cc:343] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:16.877843  1230 tablet_bootstrap.cc:654] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.881872  1234 raft_consensus.cc:357] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.882532  1234 raft_consensus.cc:383] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.885217  1234 raft_consensus.cc:738] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.885923  1234 consensus_queue.cc:260] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.893798  1234 ts_tablet_manager.cc:1428] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.030s	user 0.005s	sys 0.001s
I20250628 07:52:16.894590  1234 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:16.898741  1446 raft_consensus.cc:491] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.899367  1230 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.899170  1446 raft_consensus.cc:513] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.899710  1230 ts_tablet_manager.cc:1397] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.036s	user 0.008s	sys 0.000s
I20250628 07:52:16.900981  1446 leader_election.cc:290] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:16.902006   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c84f2570b2da45d0b3ce6428723263a5" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:16.901955  1230 raft_consensus.cc:357] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.902622  1230 raft_consensus.cc:383] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.902891  1230 raft_consensus.cc:738] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:16.903347  1062 leader_election.cc:343] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.903430  1230 consensus_queue.cc:260] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.904619   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c84f2570b2da45d0b3ce6428723263a5" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:16.906893  1062 leader_election.cc:343] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:16.907295  1062 leader_election.cc:304] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:16.907994  1446 raft_consensus.cc:2747] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:16.910238  1229 tablet_bootstrap.cc:492] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:16.910661  1229 ts_tablet_manager.cc:1397] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.103s	user 0.005s	sys 0.009s
I20250628 07:52:16.912899  1229 raft_consensus.cc:357] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.913523  1229 raft_consensus.cc:383] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.913765  1229 raft_consensus.cc:738] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.914404  1229 consensus_queue.cc:260] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.918584  1402 ts_tablet_manager.cc:1918] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:16.919658   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_17cd0eb6b1c84e65bccdd4caf5e41103"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:16.921221   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:16.922780  1229 ts_tablet_manager.cc:1428] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.012s	user 0.002s	sys 0.003s
W20250628 07:52:16.922871  1384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:52:16.923622  1384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:52:16.924379  1384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20250628 07:52:16.924780  1402 log.cc:1199] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/deb5a9ae7ab5463cae35ff1086fd8348
I20250628 07:52:16.905576  1230 ts_tablet_manager.cc:1428] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
I20250628 07:52:16.925002   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_54b2cdede05f442cb652727e32c46009" } new_table_name: "default.Table_48e99a2ce8f9404cb2bd12526a3f2f0e" modify_external_catalogs: true
I20250628 07:52:16.925778  1229 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:16.925972  1230 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.930123  1234 tablet_bootstrap.cc:654] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.933123  1402 ts_tablet_manager.cc:1939] T deb5a9ae7ab5463cae35ff1086fd8348 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:16.935245  1402 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 95) took 1048 ms. Trace:
I20250628 07:52:16.939770  1229 tablet_bootstrap.cc:654] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.940107  1402 rpcz_store.cc:276] 0628 07:52:15.886492 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.886659 (+   167us) service_pool.cc:225] Handling call
0628 07:52:16.839565 (+952906us) ts_tablet_manager.cc:1145] Deleting tablet deb5a9ae7ab5463cae35ff1086fd8348
0628 07:52:16.839612 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:16.886640 (+ 47028us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:16.918314 (+ 31674us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:16.935188 (+ 16874us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":952271}
I20250628 07:52:16.941308  1230 tablet_bootstrap.cc:654] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.941988   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet deb5a9ae7ab5463cae35ff1086fd8348 (table default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]) successfully deleted
I20250628 07:52:16.944747  1402 tablet_replica.cc:331] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:16.945669  1402 raft_consensus.cc:2241] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:16.946208  1402 raft_consensus.cc:2270] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:16.947871  1402 ts_tablet_manager.cc:1905] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:16.955433  1234 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:16.955827  1234 ts_tablet_manager.cc:1397] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.061s	user 0.006s	sys 0.002s
I20250628 07:52:16.960569  1229 tablet_bootstrap.cc:492] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:16.960963  1229 ts_tablet_manager.cc:1397] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.035s	user 0.009s	sys 0.003s
I20250628 07:52:16.966645  1230 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:16.967046  1230 ts_tablet_manager.cc:1397] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.041s	user 0.008s	sys 0.002s
I20250628 07:52:16.967272  1397 raft_consensus.cc:491] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:16.967707  1397 raft_consensus.cc:513] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.969532  1397 leader_election.cc:290] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:16.969561  1230 raft_consensus.cc:357] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.970291  1230 raft_consensus.cc:383] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.970567  1230 raft_consensus.cc:738] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.970932   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:16.971204  1230 consensus_queue.cc:260] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.972656  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:16.973240  1127 raft_consensus.cc:2466] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:16.973264  1230 ts_tablet_manager.cc:1428] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20250628 07:52:16.974522   758 leader_election.cc:304] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:16.975194  1230 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:16.975524  1397 raft_consensus.cc:2802] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:16.975879  1397 raft_consensus.cc:491] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:16.976195  1397 raft_consensus.cc:3058] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:16.963063  1229 raft_consensus.cc:357] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.976998  1234 raft_consensus.cc:357] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.981958  1234 raft_consensus.cc:383] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.982041  1229 raft_consensus.cc:383] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:16.982226  1234 raft_consensus.cc:738] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.982519  1229 raft_consensus.cc:738] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:16.982868  1234 consensus_queue.cc:260] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.983157  1229 consensus_queue.cc:260] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:16.984058   985 raft_consensus.cc:2466] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:16.988839  1230 tablet_bootstrap.cc:654] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:16.994774  1229 ts_tablet_manager.cc:1428] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.034s	user 0.004s	sys 0.000s
I20250628 07:52:16.996840  1397 raft_consensus.cc:513] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.005506  1229 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:17.007157  1402 ts_tablet_manager.cc:1918] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:17.007649  1402 log.cc:1199] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:17.008522  1402 ts_tablet_manager.cc:1939] T 67a81eb4342740e5bc16ee7f5eacb489 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.010658  1402 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 96) took 1096 ms. Trace:
I20250628 07:52:17.012763  1230 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:17.013340  1230 ts_tablet_manager.cc:1397] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.038s	user 0.010s	sys 0.000s
I20250628 07:52:17.015662  1230 raft_consensus.cc:357] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.016041   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:17.016366  1230 raft_consensus.cc:383] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.016619  1230 raft_consensus.cc:738] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.017133  1402 rpcz_store.cc:276] 0628 07:52:15.914131 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.914251 (+   120us) service_pool.cc:225] Handling call
0628 07:52:16.944569 (+1030318us) ts_tablet_manager.cc:1145] Deleting tablet 67a81eb4342740e5bc16ee7f5eacb489
0628 07:52:16.944618 (+    49us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:16.964188 (+ 19570us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.006874 (+ 42686us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.010571 (+  3697us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1029798}
I20250628 07:52:17.019812   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 67a81eb4342740e5bc16ee7f5eacb489 (table default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]) successfully deleted
I20250628 07:52:17.018559  1230 consensus_queue.cc:260] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.016682   985 raft_consensus.cc:3058] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.021468  1402 tablet_replica.cc:331] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.022320  1402 raft_consensus.cc:2241] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.028982   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b86a1653daf740d9a4927425c711c0b2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:17.029395  1402 raft_consensus.cc:2270] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:17.030541   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b86a1653daf740d9a4927425c711c0b2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:17.031252  1402 ts_tablet_manager.cc:1905] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.031141  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "474cec4459954e10a18de031946a2346" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:17.031842  1127 raft_consensus.cc:3058] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.023334  1230 ts_tablet_manager.cc:1428] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.000s
I20250628 07:52:17.037415  1446 raft_consensus.cc:491] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.037823  1446 raft_consensus.cc:513] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.039505  1446 leader_election.cc:290] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.040100  1230 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:17.041425   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:17.042904  1062 leader_election.cc:343] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.044345   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:17.045241  1230 tablet_bootstrap.cc:654] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:17.045742  1062 leader_election.cc:343] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.046130  1062 leader_election.cc:304] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.050244  1446 raft_consensus.cc:2747] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:17.049719  1127 raft_consensus.cc:2466] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.051822   758 leader_election.cc:304] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:17.052246   542 master_service.cc:948] ReplaceTablet: received request to replace tablet 84a8065c2d1349b792308ac3aec1a8fe from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:17.063585  1457 raft_consensus.cc:2802] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:17.066289  1229 tablet_bootstrap.cc:654] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.067955  1457 raft_consensus.cc:695] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:17.028751  1397 leader_election.cc:290] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:17.070359  1234 ts_tablet_manager.cc:1428] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.114s	user 0.011s	sys 0.011s
I20250628 07:52:17.073406  1234 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:17.072305  1457 consensus_queue.cc:237] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.066660   985 raft_consensus.cc:2466] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.079950  1459 consensus_queue.cc:1035] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:17.082198  1234 tablet_bootstrap.cc:654] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.103911  1234 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:17.111754  1230 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:17.112154  1230 ts_tablet_manager.cc:1397] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.074s	user 0.010s	sys 0.000s
I20250628 07:52:17.110982  1397 consensus_queue.cc:1035] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:17.114964  1230 raft_consensus.cc:357] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.115581  1230 raft_consensus.cc:383] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.115846  1230 raft_consensus.cc:738] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.116413  1230 consensus_queue.cc:260] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.118484  1230 ts_tablet_manager.cc:1428] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20250628 07:52:17.118839  1402 ts_tablet_manager.cc:1918] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:17.119171  1402 log.cc:1199] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/0f22700fc70d4f14ae51a834900db22b
I20250628 07:52:17.119829  1402 ts_tablet_manager.cc:1939] T 0f22700fc70d4f14ae51a834900db22b P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
W20250628 07:52:17.122238   534 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 75884.5us
I20250628 07:52:17.124481  1230 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:17.125593  1446 raft_consensus.cc:491] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.125978  1446 raft_consensus.cc:513] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.127820  1446 leader_election.cc:290] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.129602   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:17.130461   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.131078  1062 leader_election.cc:343] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:17.131752  1062 leader_election.cc:343] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.132159  1062 leader_election.cc:304] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.133366  1446 raft_consensus.cc:2747] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:17.134536  1244 ts_tablet_manager.cc:1918] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.134873  1244 log.cc:1199] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/3637ab35a2b74ce3a111c77cbec8b0b0
I20250628 07:52:17.135553  1244 ts_tablet_manager.cc:1939] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:17.144404  1229 tablet_bootstrap.cc:492] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:17.144783  1229 ts_tablet_manager.cc:1397] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.140s	user 0.009s	sys 0.000s
I20250628 07:52:17.146960  1229 raft_consensus.cc:357] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.147549  1229 raft_consensus.cc:383] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.147783  1229 raft_consensus.cc:738] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.149223  1446 raft_consensus.cc:491] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.149724  1446 raft_consensus.cc:513] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.151660  1446 leader_election.cc:290] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.154707  1402 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 98) took 1155 ms. Trace:
I20250628 07:52:17.155025  1402 rpcz_store.cc:276] 0628 07:52:15.999583 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.999941 (+   358us) service_pool.cc:225] Handling call
0628 07:52:17.021318 (+1021377us) ts_tablet_manager.cc:1145] Deleting tablet 0f22700fc70d4f14ae51a834900db22b
0628 07:52:17.021362 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:17.075337 (+ 53975us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.118623 (+ 43286us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.154626 (+ 36003us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1020821}
I20250628 07:52:17.156121   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 0f22700fc70d4f14ae51a834900db22b (table default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]) successfully deleted
I20250628 07:52:17.164265   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3223005021ae4e38b4ac65af460d0ad0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.165369  1062 leader_election.cc:343] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.173491  1402 tablet_replica.cc:331] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.174326  1402 raft_consensus.cc:2241] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.174820  1402 raft_consensus.cc:2270] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.181341  1402 ts_tablet_manager.cc:1905] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.185812   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111), 
I20250628 07:52:17.186213   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:17.141220  1234 ts_tablet_manager.cc:1397] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.068s	user 0.011s	sys 0.000s
I20250628 07:52:17.144193  1397 raft_consensus.cc:491] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.197185  1397 raft_consensus.cc:513] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.199198  1397 leader_election.cc:290] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:17.153242   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3223005021ae4e38b4ac65af460d0ad0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:17.198936  1234 raft_consensus.cc:357] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.201256  1234 raft_consensus.cc:383] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.201205  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:17.201537  1234 raft_consensus.cc:738] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.201733  1127 raft_consensus.cc:2466] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:17.201907  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 77) took 1361 ms. Trace:
I20250628 07:52:17.202289  1234 consensus_queue.cc:260] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.202752   758 leader_election.cc:304] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:17.203141   824 raft_consensus.cc:2466] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:17.203161  1229 consensus_queue.cc:260] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:17.205631   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 16a8bee148664e948529d32c0cf2c793 consensus state has no leader
I20250628 07:52:17.206364   547 master_service.cc:948] ReplaceTablet: received request to replace tablet d62ecf32bcc54310b9114d5655be7673 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:17.207468  1062 leader_election.cc:304] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.208142  1397 raft_consensus.cc:2802] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:17.208433  1446 raft_consensus.cc:2802] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:17.208453  1397 raft_consensus.cc:491] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:17.208748  1446 raft_consensus.cc:491] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:17.208777  1397 raft_consensus.cc:3058] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.209010  1446 raft_consensus.cc:3058] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.215373  1446 raft_consensus.cc:513] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.217427  1230 tablet_bootstrap.cc:654] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.173346  1457 raft_consensus.cc:491] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.218108  1457 raft_consensus.cc:513] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.218333   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3223005021ae4e38b4ac65af460d0ad0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:17.220247   824 raft_consensus.cc:3058] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.221853  1457 leader_election.cc:290] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:17.226845   758 leader_election.cc:343] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:17.228716  1234 ts_tablet_manager.cc:1428] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.004s	sys 0.000s
I20250628 07:52:17.209087  1229 ts_tablet_manager.cc:1428] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.064s	user 0.006s	sys 0.000s
I20250628 07:52:17.231513  1234 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:17.236240  1229 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:17.242311   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3223005021ae4e38b4ac65af460d0ad0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:17.244073  1062 leader_election.cc:343] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.244623  1446 leader_election.cc:290] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.247037   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0ef22eef86e5432bb09a3f878ed94458" } modify_external_catalogs: true
I20250628 07:52:17.247555   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0ef22eef86e5432bb09a3f878ed94458" } modify_external_catalogs: true
I20250628 07:52:17.216485  1383 ts_tablet_manager.cc:1918] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.221580   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67a81eb4342740e5bc16ee7f5eacb489" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:17.253052   984 raft_consensus.cc:2466] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:17.217823   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:17.254236   757 leader_election.cc:304] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:17.255663   985 raft_consensus.cc:2466] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:17.256134  1457 raft_consensus.cc:2802] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:17.256479  1457 raft_consensus.cc:491] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:17.256809  1457 raft_consensus.cc:3058] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.258545  1383 log.cc:1199] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/3637ab35a2b74ce3a111c77cbec8b0b0
I20250628 07:52:17.260179  1383 ts_tablet_manager.cc:1939] T 3637ab35a2b74ce3a111c77cbec8b0b0 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:17.216967  1397 raft_consensus.cc:513] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.280952  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:17.281534  1128 raft_consensus.cc:3058] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.284231   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1642348a16574345b31cebda23575102" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:17.284503   824 raft_consensus.cc:2466] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:17.286118   985 raft_consensus.cc:3058] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.288323  1128 raft_consensus.cc:2466] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.289213  1062 leader_election.cc:304] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.289981  1446 raft_consensus.cc:2802] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:17.290405  1446 raft_consensus.cc:695] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:17.290521   758 leader_election.cc:304] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:17.291409  1234 tablet_bootstrap.cc:654] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.291333  1455 raft_consensus.cc:2802] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:17.291167  1446 consensus_queue.cc:237] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.296644   985 raft_consensus.cc:2466] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.304804  1230 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:17.305193  1230 ts_tablet_manager.cc:1397] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.181s	user 0.011s	sys 0.001s
I20250628 07:52:17.307190  1230 raft_consensus.cc:357] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.307803  1230 raft_consensus.cc:383] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.308043  1230 raft_consensus.cc:738] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.308827  1455 raft_consensus.cc:695] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:17.264470  1229 tablet_bootstrap.cc:654] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.309607  1455 consensus_queue.cc:237] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.271626  1383 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 74) took 1430 ms. Trace:
I20250628 07:52:17.312316  1383 rpcz_store.cc:276] 0628 07:52:15.841415 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.842698 (+  1283us) service_pool.cc:225] Handling call
0628 07:52:16.623783 (+781085us) ts_tablet_manager.cc:1145] Deleting tablet 3637ab35a2b74ce3a111c77cbec8b0b0
0628 07:52:16.623839 (+    56us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:16.868288 (+244449us) spinlock_profiling.cc:243] Waited 236 ms on lock 0x7b58000e00b0. stack: 00007ff6ffd094b3 00007ff6ffd093b0 00007ff6ff885574 00000000004fa174 00000000004fa109 00007ff706c0b5d1 00007ff706c01ef3 00007ff706c652f7 00007ff706c8f0b5 00007ff7070ae62c 00007ff707cc1d1a 00007ff707cf1b70 00007ff707cd8f4d 00007ff707cd8ef9 00007ff707cd8e89 00007ff707cd8e51
0628 07:52:17.064472 (+196184us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.216255 (+151783us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.271504 (+ 55249us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":542227712,"tablet-delete.queue_time_us":780560}
I20250628 07:52:17.314196   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (table default.Table_2d6aec0ecd124912b4fdcea46bff2243 [id=853998210b39471f9d2d0bf0ebf072d5]) successfully deleted
I20250628 07:52:17.319094   965 tablet_service.cc:1468] Processing CreateTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (DEFAULT_TABLE table=default.Table_f892200f501c448a90964969f585547c [id=398556bc3bdc481cb6577321cbc31055]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.320192   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.324760  1402 ts_tablet_manager.cc:1918] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:17.332641   802 tablet_service.cc:1468] Processing CreateTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (DEFAULT_TABLE table=default.Table_f892200f501c448a90964969f585547c [id=398556bc3bdc481cb6577321cbc31055]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:17.333701  1384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
I20250628 07:52:17.333760   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:17.334066  1384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:52:17.334378  1384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:52:17.334684  1384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:52:17.342521  1234 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:17.342945  1234 ts_tablet_manager.cc:1397] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.113s	user 0.008s	sys 0.000s
I20250628 07:52:17.347891  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (DEFAULT_TABLE table=default.Table_f892200f501c448a90964969f585547c [id=398556bc3bdc481cb6577321cbc31055]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.348991  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.349542   803 tablet_service.cc:1468] Processing CreateTablet for tablet 60179e6765b145adbf637a8435eef724 (DEFAULT_TABLE table=default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.350492   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 60179e6765b145adbf637a8435eef724. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.351604  1446 raft_consensus.cc:491] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.352030  1446 raft_consensus.cc:513] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.353681  1446 leader_election.cc:290] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.354601  1230 consensus_queue.cc:260] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.356439  1230 ts_tablet_manager.cc:1428] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.051s	user 0.003s	sys 0.000s
I20250628 07:52:17.359045  1230 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:17.361915  1234 raft_consensus.cc:357] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.362527  1234 raft_consensus.cc:383] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.362749  1234 raft_consensus.cc:738] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.363353  1234 consensus_queue.cc:260] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.365134  1234 ts_tablet_manager.cc:1428] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.022s	user 0.007s	sys 0.000s
I20250628 07:52:17.366091  1402 log.cc:1199] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/474cec4459954e10a18de031946a2346
I20250628 07:52:17.367072  1402 ts_tablet_manager.cc:1939] T 474cec4459954e10a18de031946a2346 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.368984  1402 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 97) took 1370 ms. Trace:
I20250628 07:52:17.369314  1402 rpcz_store.cc:276] 0628 07:52:15.998631 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:16.001142 (+  2511us) service_pool.cc:225] Handling call
0628 07:52:17.173333 (+1172191us) ts_tablet_manager.cc:1145] Deleting tablet 474cec4459954e10a18de031946a2346
0628 07:52:17.173377 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:17.281182 (+107805us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.324529 (+ 43347us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.368922 (+ 44393us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1171462}
I20250628 07:52:17.370786  1402 tablet_replica.cc:331] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.371228   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 474cec4459954e10a18de031946a2346 (table default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]) successfully deleted
I20250628 07:52:17.371526  1402 raft_consensus.cc:2241] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.371484  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 60179e6765b145adbf637a8435eef724 (DEFAULT_TABLE table=default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:17.371922   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet ef449bd95fc64bf28c854b03a219c7b4 because the tablet was not found. No further retry: Not found: Tablet not found: ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:17.372056  1402 raft_consensus.cc:2270] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.372625  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 60179e6765b145adbf637a8435eef724. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:17.384455   758 consensus_peers.cc:489] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:17.374418   805 tablet_service.cc:1468] Processing CreateTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.385859   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.390695   962 tablet_service.cc:1468] Processing CreateTablet for tablet 60179e6765b145adbf637a8435eef724 (DEFAULT_TABLE table=default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.391846   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 60179e6765b145adbf637a8435eef724. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.395884  1230 tablet_bootstrap.cc:654] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.384673  1234 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:17.397418   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.398742  1062 leader_election.cc:343] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.202337  1244 rpcz_store.cc:276] 0628 07:52:15.840234 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.841951 (+  1717us) service_pool.cc:225] Handling call
0628 07:52:16.529685 (+687734us) ts_tablet_manager.cc:1145] Deleting tablet 3637ab35a2b74ce3a111c77cbec8b0b0
0628 07:52:16.529735 (+    50us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:16.943812 (+414077us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.134319 (+190507us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:17.201846 (+ 67527us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":687167}
I20250628 07:52:17.414541  1402 ts_tablet_manager.cc:1905] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.418032   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 3637ab35a2b74ce3a111c77cbec8b0b0 (table default.Table_2d6aec0ecd124912b4fdcea46bff2243 [id=853998210b39471f9d2d0bf0ebf072d5]) successfully deleted
I20250628 07:52:17.407825  1397 leader_election.cc:290] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:17.376740   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:17.420835  1062 leader_election.cc:343] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.421262  1062 leader_election.cc:304] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.421885  1446 raft_consensus.cc:2747] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:17.422158  1235 raft_consensus.cc:491] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.422535  1235 raft_consensus.cc:513] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.425175  1234 tablet_bootstrap.cc:654] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.433831   965 tablet_service.cc:1468] Processing CreateTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.434854   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.440762  1446 raft_consensus.cc:491] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.441388  1446 raft_consensus.cc:513] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.443176  1446 leader_election.cc:290] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.444008  1235 leader_election.cc:290] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.445781   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d39d14572c4647c7b4880c71a8707c08" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:17.451709   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:17.453042   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d39d14572c4647c7b4880c71a8707c08" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.456406  1062 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.454682   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:17.458072  1229 tablet_bootstrap.cc:492] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:17.458631  1229 ts_tablet_manager.cc:1397] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.224s	user 0.010s	sys 0.002s
W20250628 07:52:17.459185  1062 leader_election.cc:343] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:17.460485  1062 leader_election.cc:343] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.461964  1062 leader_election.cc:304] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.463346  1446 raft_consensus.cc:2747] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:17.464432  1062 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.467453  1234 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:17.467870  1234 ts_tablet_manager.cc:1397] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.083s	user 0.008s	sys 0.002s
I20250628 07:52:17.467036  1230 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:17.468410  1230 ts_tablet_manager.cc:1397] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.110s	user 0.009s	sys 0.000s
I20250628 07:52:17.470239  1234 raft_consensus.cc:357] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.470904  1234 raft_consensus.cc:383] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.471158  1234 raft_consensus.cc:738] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.471366  1230 raft_consensus.cc:357] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.479357  1230 raft_consensus.cc:383] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.483181   964 tablet_service.cc:1468] Processing CreateTablet for tablet e15d063a4bca40d480cba08733ec1ec3 (DEFAULT_TABLE table=default.Table_6da92141ad354b5491ebafee908fe6d7 [id=475e75d866ea4a5585227052d39db9da]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.484328   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e15d063a4bca40d480cba08733ec1ec3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.483064  1229 raft_consensus.cc:357] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.493208  1229 raft_consensus.cc:383] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.472175  1062 leader_election.cc:304] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.493503  1229 raft_consensus.cc:738] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.492527  1402 ts_tablet_manager.cc:1918] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:17.493937  1446 raft_consensus.cc:2747] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:17.494118  1229 consensus_queue.cc:260] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.495738  1230 raft_consensus.cc:738] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.499867  1108 tablet_service.cc:1468] Processing CreateTablet for tablet e15d063a4bca40d480cba08733ec1ec3 (DEFAULT_TABLE table=default.Table_6da92141ad354b5491ebafee908fe6d7 [id=475e75d866ea4a5585227052d39db9da]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.500789  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e15d063a4bca40d480cba08733ec1ec3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.501605  1229 ts_tablet_manager.cc:1428] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.043s	user 0.001s	sys 0.005s
I20250628 07:52:17.502084  1402 log.cc:1199] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:17.502697  1402 ts_tablet_manager.cc:1939] T fa96605dbabe4f0fbc26b057232960a4 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.504556  1229 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:17.504632   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fa96605dbabe4f0fbc26b057232960a4 (table default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]) successfully deleted
I20250628 07:52:17.505472  1402 tablet_replica.cc:331] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.506139  1402 raft_consensus.cc:2241] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.506513  1402 raft_consensus.cc:2270] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.507966  1402 ts_tablet_manager.cc:1905] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.471719  1234 consensus_queue.cc:260] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.515071  1234 ts_tablet_manager.cc:1428] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.047s	user 0.005s	sys 0.000s
I20250628 07:52:17.515884  1234 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:17.516449   801 tablet_service.cc:1468] Processing CreateTablet for tablet e15d063a4bca40d480cba08733ec1ec3 (DEFAULT_TABLE table=default.Table_6da92141ad354b5491ebafee908fe6d7 [id=475e75d866ea4a5585227052d39db9da]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.517601  1477 consensus_queue.cc:1035] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:17.517607   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e15d063a4bca40d480cba08733ec1ec3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.531929  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (DEFAULT_TABLE table=default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.532564  1384 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:33331
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:37209,127.0.107.189:33023,127.0.107.188:42677
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:37209
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.107.190
--webserver_port=42501
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:52:17.533032  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.534248  1384 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:52:17.536312  1384 file_cache.cc:492] Constructed file cache file cache with capacity 419430
I20250628 07:52:17.543851  1457 raft_consensus.cc:513] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:17.546041   758 consensus_peers.cc:489] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f22700fc70d4f14ae51a834900db22b. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:17.496871  1230 consensus_queue.cc:260] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.569190  1234 tablet_bootstrap.cc:654] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.570904  1230 ts_tablet_manager.cc:1428] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.102s	user 0.005s	sys 0.000s
I20250628 07:52:17.574332  1457 leader_election.cc:290] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:17.576565  1486 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:52:17.589490   758 leader_election.cc:343] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:17.592566   962 tablet_service.cc:1468] Processing CreateTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.594555  1229 tablet_bootstrap.cc:654] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.596391   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67a81eb4342740e5bc16ee7f5eacb489" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:17.596940   984 raft_consensus.cc:3058] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.606545   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01feb8ef40dc4707bd6251fc2d87f46f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.630896   984 raft_consensus.cc:2466] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.632211   757 leader_election.cc:304] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:17.633232  1397 raft_consensus.cc:2802] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:17.633622  1397 raft_consensus.cc:695] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:17.634325  1397 consensus_queue.cc:237] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:17.635735   921 consensus_peers.cc:489] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: deb5a9ae7ab5463cae35ff1086fd8348. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:17.635663   805 tablet_service.cc:1468] Processing CreateTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.636587   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01feb8ef40dc4707bd6251fc2d87f46f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.576810  1402 ts_tablet_manager.cc:1918] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:17.633622  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.638355  1402 log.cc:1199] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:17.639194  1402 ts_tablet_manager.cc:1939] T 2b51d6076e344c8f8e388b77b4b23928 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.639312  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01feb8ef40dc4707bd6251fc2d87f46f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.642113   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 2b51d6076e344c8f8e388b77b4b23928 (table default.Table_c22b7ab2180346e093151880dbd4cb83 [id=3602ce86ce23434d83a8641b502843c4]) successfully deleted
I20250628 07:52:17.670830   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 84a8065c2d1349b792308ac3aec1a8fe
I20250628 07:52:17.672348   542 catalog_manager.cc:6652] ReplaceTablet: tablet 84a8065c2d1349b792308ac3aec1a8fe deleted and replaced by tablet ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:17.678809  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.679554  1402 tablet_replica.cc:331] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.680219  1402 raft_consensus.cc:2241] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.680624  1402 raft_consensus.cc:2270] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.682225  1402 ts_tablet_manager.cc:1905] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.683058  1477 raft_consensus.cc:491] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.683465  1477 raft_consensus.cc:513] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.685220  1477 leader_election.cc:290] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:17.676785   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:17.685081   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 29acadb819ef46318e2ba5ba2d3c3c37
I20250628 07:52:17.689119   545 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 132) took 1644 ms. Trace:
I20250628 07:52:17.690562   986 raft_consensus.cc:3058] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.694445   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
I20250628 07:52:17.695709  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.697742   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.698449  1383 tablet_replica.cc:331] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:17.699193  1383 raft_consensus.cc:2241] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.699632  1383 raft_consensus.cc:2270] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.701694   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.701712  1383 ts_tablet_manager.cc:1905] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.702158   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.701853  1457 consensus_queue.cc:1035] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:17.703235   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:17.703920   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 1 got 0
I20250628 07:52:17.704571   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_a0e49295ed0e482eaa16a7761b01b32c [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 1 got 0
W20250628 07:52:17.705339   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 29acadb819ef46318e2ba5ba2d3c3c37 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29acadb819ef46318e2ba5ba2d3c3c37 already in progress: creating tablet
I20250628 07:52:17.686455   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:17.707953   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:17.711681   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.712837   757 leader_election.cc:343] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.689463   545 rpcz_store.cc:276] 0628 07:52:16.044642 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:16.044771 (+   129us) service_pool.cc:225] Handling call
0628 07:52:16.046015 (+  1244us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:16.046525 (+   510us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:16.046651 (+   126us) catalog_manager.cc:2799] Locking tablets
0628 07:52:17.307120 (+1260469us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:17.684733 (+377613us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:17.684916 (+   183us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:17.685002 (+    86us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:17.688994 (+  3992us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:17.474762 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:17.475120 (+   358us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:17.475140 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:17.475152 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:17.475422 (+   270us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:17.475454 (+    32us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:17.475476 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:17.475716 (+   240us) tablet.cc:699] Row locks acquired
0628 07:52:17.475727 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:17.475827 (+   100us) write_op.cc:270] Start()
0628 07:52:17.475923 (+    96us) write_op.cc:276] Timestamp: P: 1751097137475777 usec, L: 0
0628 07:52:17.475939 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:17.476357 (+   418us) log.cc:844] Serialized 731 byte log entry
0628 07:52:17.680445 (+204088us) op_driver.cc:464] REPLICATION: finished
0628 07:52:17.683575 (+  3130us) write_op.cc:301] APPLY: starting
0628 07:52:17.683643 (+    68us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:17.683731 (+    88us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:17.683800 (+    69us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:17.684006 (+   206us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:17.684122 (+   116us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:17.684161 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:17.684316 (+   155us) log.cc:844] Serialized 29 byte log entry
0628 07:52:17.684442 (+   126us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:17.684500 (+    58us) write_op.cc:454] Released schema lock
0628 07:52:17.684547 (+    47us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:17.685486 (+   939us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:17.685515 (+    29us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2975,"apply.run_cpu_time_us":1198,"apply.run_wall_time_us":1874,"num_ops":2,"prepare.queue_time_us":166105,"prepare.run_cpu_time_us":1915,"prepare.run_wall_time_us":93065,"raft.queue_time_us":1764,"raft.run_cpu_time_us":13,"raft.run_wall_time_us":14,"replication_time_us":204400}]]}
I20250628 07:52:17.717669   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7c51004dd5724c6499326369c8712f8a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:17.719660   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7c51004dd5724c6499326369c8712f8a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:17.724363  1402 ts_tablet_manager.cc:1918] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.731791  1383 ts_tablet_manager.cc:1918] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.734602  1402 log.cc:1199] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/84a8065c2d1349b792308ac3aec1a8fe
I20250628 07:52:17.735659  1402 ts_tablet_manager.cc:1939] T 84a8065c2d1349b792308ac3aec1a8fe P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.738574   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
I20250628 07:52:17.739719  1383 log.cc:1199] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/84a8065c2d1349b792308ac3aec1a8fe
I20250628 07:52:17.740880  1383 ts_tablet_manager.cc:1939] T 84a8065c2d1349b792308ac3aec1a8fe P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:17.742852  1234 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:17.743304  1234 ts_tablet_manager.cc:1397] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.228s	user 0.010s	sys 0.001s
W20250628 07:52:17.744371   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 29acadb819ef46318e2ba5ba2d3c3c37 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29acadb819ef46318e2ba5ba2d3c3c37 already in progress: creating tablet
I20250628 07:52:17.747135   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_ed752c51c98743c6b11f0f3a2ce3db69 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
W20250628 07:52:17.751024   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 29acadb819ef46318e2ba5ba2d3c3c37 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29acadb819ef46318e2ba5ba2d3c3c37 already in progress: creating tablet
I20250628 07:52:17.751612  1457 raft_consensus.cc:491] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.753049  1230 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:17.752025  1457 raft_consensus.cc:513] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:17.755671   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3], current schema version=3) failed: Not found: failed to reset TS proxy: No leader found: Tablet 2d77be1f37d2400f842c01019e4eb474 consensus state has no leader
I20250628 07:52:17.755645  1457 leader_election.cc:290] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:17.756700   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2df135033ee546f8bb9f4c4b54755b06" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:17.757292   985 raft_consensus.cc:2466] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:17.758334   757 leader_election.cc:304] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:17.739436  1229 tablet_bootstrap.cc:492] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:17.764977  1229 ts_tablet_manager.cc:1397] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.261s	user 0.011s	sys 0.000s
I20250628 07:52:17.770562   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:17.771248  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:17.771849   547 catalog_manager.cc:6652] ReplaceTablet: tablet d62ecf32bcc54310b9114d5655be7673 deleted and replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:17.771835  1128 raft_consensus.cc:2466] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:17.774057  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.774034   803 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.775219   965 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:17.775671   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet d62ecf32bcc54310b9114d5655be7673 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d62ecf32bcc54310b9114d5655be7673 already in progress: creating tablet
W20250628 07:52:17.776600   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d62ecf32bcc54310b9114d5655be7673 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d62ecf32bcc54310b9114d5655be7673 already in progress: creating tablet
W20250628 07:52:17.776742   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d62ecf32bcc54310b9114d5655be7673 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d62ecf32bcc54310b9114d5655be7673 already in progress: creating tablet
I20250628 07:52:17.779637  1457 raft_consensus.cc:2802] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:17.779996  1457 raft_consensus.cc:491] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:17.780457  1457 raft_consensus.cc:3058] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.783365  1230 tablet_bootstrap.cc:654] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.772675   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c0546593fb7742e3b88d4b60b53ca658"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:17.787314   758 leader_election.cc:304] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:17.788846   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c0546593fb7742e3b88d4b60b53ca658 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:17.791086   758 leader_election.cc:343] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:17.793146  1455 raft_consensus.cc:2802] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:17.793489  1455 raft_consensus.cc:491] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:17.793772  1455 raft_consensus.cc:3058] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:17.811748   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:17.821261   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:17.823938   546 catalog_manager.cc:5582] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:17.827893   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
I20250628 07:52:17.830008   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:17.833415  1446 raft_consensus.cc:491] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:17.833894  1446 raft_consensus.cc:513] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.834932  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.835533  1402 tablet_replica.cc:331] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.835682  1446 leader_election.cc:290] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:17.836182  1402 raft_consensus.cc:2241] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.836594  1402 raft_consensus.cc:2270] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.836609   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.837919  1244 tablet_replica.cc:331] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:17.838201  1402 ts_tablet_manager.cc:1905] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.838676  1244 raft_consensus.cc:2241] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.839123  1244 raft_consensus.cc:2270] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.841121  1244 ts_tablet_manager.cc:1905] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.841269   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:17.842578   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet ef060774191146168ee8d1e13c9cd71b has no consensus state
I20250628 07:52:17.842598  1383 tablet_replica.cc:331] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:17.843027   540 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 129) took 2093 ms. Trace:
I20250628 07:52:17.843619  1383 raft_consensus.cc:2241] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:17.844693  1383 raft_consensus.cc:2270] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.847215  1383 ts_tablet_manager.cc:1905] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.847292   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.847987   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:17.847127   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:17.863617  1062 leader_election.cc:343] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:17.864255  1062 leader_election.cc:343] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:17.864610  1062 leader_election.cc:304] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:17.865574  1234 raft_consensus.cc:357] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.769404  1229 raft_consensus.cc:357] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.869302  1229 raft_consensus.cc:383] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.869577  1229 raft_consensus.cc:738] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.870172  1229 consensus_queue.cc:260] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.866187  1234 raft_consensus.cc:383] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:17.876817  1234 raft_consensus.cc:738] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:17.877437  1234 consensus_queue.cc:260] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.879402  1234 ts_tablet_manager.cc:1428] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.136s	user 0.006s	sys 0.000s
I20250628 07:52:17.881537   964 tablet_service.cc:1468] Processing CreateTablet for tablet fab31b073e2e49318b4118851f81b6d2 (DEFAULT_TABLE table=default.Table_4a45d5758c0f40209ba04b7d96752a10 [id=491d07dd82d54928b39605ba6d28af42]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.882782  1402 ts_tablet_manager.cc:1918] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.884011   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fab31b073e2e49318b4118851f81b6d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.888159  1402 log.cc:1199] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:17.887207   965 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:17.888350   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 31) took 2705 ms (client timeout 1000 ms). Trace:
I20250628 07:52:17.889454   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.890725   966 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.891530  1402 ts_tablet_manager.cc:1939] T 2f7e2b290c69467aa42ca6328307bc3c P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:17.891444   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.843287   540 rpcz_store.cc:276] 0628 07:52:15.749774 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.749868 (+    94us) service_pool.cc:225] Handling call
0628 07:52:15.929340 (+179472us) spinlock_profiling.cc:243] Waited 172 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea9413b6 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc 00007f5be6957206
0628 07:52:15.930025 (+   685us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:15.934055 (+  4030us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:15.934557 (+   502us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:15.934703 (+   146us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:16.549946 (+615243us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:17.842967 (+1293021us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:16.370495 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:16.371178 (+   683us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:16.371211 (+    33us) write_op.cc:435] Acquired schema lock
0628 07:52:16.371230 (+    19us) tablet.cc:660] Decoding operations
0628 07:52:16.371536 (+   306us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:16.371584 (+    48us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:16.371618 (+    34us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:16.371810 (+   192us) tablet.cc:699] Row locks acquired
0628 07:52:16.371829 (+    19us) write_op.cc:260] PREPARE: finished
0628 07:52:16.371959 (+   130us) write_op.cc:270] Start()
0628 07:52:16.372056 (+    97us) write_op.cc:276] Timestamp: P: 1751097136371907 usec, L: 0
0628 07:52:16.372078 (+    22us) op_driver.cc:348] REPLICATION: starting
0628 07:52:16.372609 (+   531us) log.cc:844] Serialized 405 byte log entry
0628 07:52:16.539781 (+167172us) op_driver.cc:464] REPLICATION: finished
0628 07:52:16.547967 (+  8186us) write_op.cc:301] APPLY: starting
0628 07:52:16.548032 (+    65us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:16.548142 (+   110us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:16.548268 (+   126us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:16.548447 (+   179us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:16.548610 (+   163us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:16.548678 (+    68us) write_op.cc:312] APPLY: finished
0628 07:52:16.548895 (+   217us) log.cc:844] Serialized 23 byte log entry
0628 07:52:16.549607 (+   712us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:16.549687 (+    80us) write_op.cc:454] Released schema lock
0628 07:52:16.549726 (+    39us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:16.550411 (+   685us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:16.550444 (+    33us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":394843008,"child_traces":[["op",{"apply.queue_time_us":7978,"apply.run_cpu_time_us":1512,"apply.run_wall_time_us":2426,"num_ops":1,"prepare.queue_time_us":432265,"prepare.run_cpu_time_us":2367,"prepare.run_wall_time_us":125368,"raft.queue_time_us":72,"raft.run_cpu_time_us":208,"raft.run_wall_time_us":208,"replication_time_us":167545,"spinlock_wait_cycles":160128}]]}
I20250628 07:52:17.897496  1446 raft_consensus.cc:2747] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:17.899348   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 2f7e2b290c69467aa42ca6328307bc3c (table default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]) successfully deleted
I20250628 07:52:17.903846   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:17.904673   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 47491.3us
I20250628 07:52:17.905386   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.906884   803 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.908243   803 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:17.893628   539 rpcz_store.cc:269] 0628 07:52:15.182910 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:15.183022 (+   112us) service_pool.cc:225] Handling call
0628 07:52:17.888271 (+2705249us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:16.644774 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:16.646362 (+  1588us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:16.646396 (+    34us) write_op.cc:435] Acquired schema lock
0628 07:52:16.646416 (+    20us) tablet.cc:660] Decoding operations
0628 07:52:16.646782 (+   366us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:16.646823 (+    41us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:16.646843 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:16.647093 (+   250us) tablet.cc:699] Row locks acquired
0628 07:52:16.647106 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:16.647197 (+    91us) write_op.cc:270] Start()
0628 07:52:16.647308 (+   111us) write_op.cc:276] Timestamp: P: 1751097136647126 usec, L: 0
0628 07:52:16.647325 (+    17us) op_driver.cc:348] REPLICATION: starting
0628 07:52:16.647789 (+   464us) log.cc:844] Serialized 913 byte log entry
0628 07:52:16.661877 (+ 14088us) op_driver.cc:464] REPLICATION: finished
0628 07:52:16.669871 (+  7994us) write_op.cc:301] APPLY: starting
0628 07:52:16.669944 (+    73us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:16.670075 (+   131us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:16.670186 (+   111us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:16.670453 (+   267us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:16.670665 (+   212us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:16.670736 (+    71us) write_op.cc:312] APPLY: finished
0628 07:52:16.670968 (+   232us) log.cc:844] Serialized 29 byte log entry
0628 07:52:16.671731 (+   763us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:16.671841 (+   110us) write_op.cc:454] Released schema lock
0628 07:52:16.671883 (+    42us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:16.754003 (+ 82120us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:16.754027 (+    24us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":8,"child_traces":[["op",{"apply.queue_time_us":7809,"apply.run_cpu_time_us":1690,"apply.run_wall_time_us":38593,"num_ops":2,"prepare.queue_time_us":647000,"prepare.run_cpu_time_us":2270,"prepare.run_wall_time_us":109222,"raft.queue_time_us":95,"raft.run_cpu_time_us":823,"raft.run_wall_time_us":58776,"replication_time_us":14447}]]}
I20250628 07:52:17.910962   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a0e49295ed0e482eaa16a7761b01b32c" } new_table_name: "default.Table_f84ba40fc1d14d55a6725054f1c3465f" modify_external_catalogs: true
I20250628 07:52:17.912114  1383 ts_tablet_manager.cc:1918] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:17.912487  1383 log.cc:1199] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:17.913262  1383 ts_tablet_manager.cc:1939] T 2f7e2b290c69467aa42ca6328307bc3c P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:17.919643  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.920635  1383 tablet_replica.cc:331] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:17.920742   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 2f7e2b290c69467aa42ca6328307bc3c (table default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]) successfully deleted
I20250628 07:52:17.913636  1504 consensus_queue.cc:1035] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
W20250628 07:52:17.921417   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ef449bd95fc64bf28c854b03a219c7b4 because the tablet was not found. No further retry: Not found: Tablet not found: ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:17.922494   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:17.922685   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 32) took 1135 ms (client timeout 1000 ms). Trace:
W20250628 07:52:17.922946   544 rpcz_store.cc:269] 0628 07:52:16.787577 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:16.787689 (+   112us) service_pool.cc:225] Handling call
0628 07:52:16.818061 (+ 30372us) spinlock_profiling.cc:243] Waited 28.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:17.922634 (+1104573us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":9,"spinlock_wait_cycles":64931584}
I20250628 07:52:17.924283   961 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.915673  1229 ts_tablet_manager.cc:1428] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.150s	user 0.005s	sys 0.000s
I20250628 07:52:17.924991  1402 tablet_replica.cc:331] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:17.925781  1229 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:17.925959  1402 raft_consensus.cc:2241] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:17.926333  1402 raft_consensus.cc:2270] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.927961   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:17.928117  1402 ts_tablet_manager.cc:1905] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:17.930251  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:17.930917  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:17.935936   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 35) took 1114 ms (client timeout 1000 ms). Trace:
W20250628 07:52:17.936271   538 rpcz_store.cc:269] 0628 07:52:16.821427 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:16.821550 (+   123us) service_pool.cc:225] Handling call
0628 07:52:16.929907 (+108357us) spinlock_profiling.cc:243] Waited 101 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:17.935875 (+1005968us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":14444,"reported_tablets":7,"spinlock_wait_cycles":232797952}
I20250628 07:52:17.938035   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c6657dd54c2f4decb3eda54e473d4c10" } modify_external_catalogs: true
I20250628 07:52:17.938650   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c6657dd54c2f4decb3eda54e473d4c10" } modify_external_catalogs: true
I20250628 07:52:17.942723   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:17.942397   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 33) took 1881 ms (client timeout 1000 ms). Trace:
W20250628 07:52:17.943363   543 rpcz_store.cc:269] 0628 07:52:16.061248 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:16.158905 (+ 97657us) service_pool.cc:225] Handling call
0628 07:52:17.942339 (+1783434us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":10}
I20250628 07:52:17.944103   802 tablet_service.cc:1515] Processing DeleteTablet for tablet ef449bd95fc64bf28c854b03a219c7b4 with delete_type TABLET_DATA_DELETED (replaced by tablet 8f7dde9b060f4d839724e5f718519fb1) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:17.947135   802 tablet_service.cc:1468] Processing CreateTablet for tablet fab31b073e2e49318b4118851f81b6d2 (DEFAULT_TABLE table=default.Table_4a45d5758c0f40209ba04b7d96752a10 [id=491d07dd82d54928b39605ba6d28af42]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.947433  1108 tablet_service.cc:1468] Processing CreateTablet for tablet fab31b073e2e49318b4118851f81b6d2 (DEFAULT_TABLE table=default.Table_4a45d5758c0f40209ba04b7d96752a10 [id=491d07dd82d54928b39605ba6d28af42]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.948486  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fab31b073e2e49318b4118851f81b6d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.949852  1383 raft_consensus.cc:2241] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:17.951761   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fab31b073e2e49318b4118851f81b6d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.954762  1229 tablet_bootstrap.cc:654] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:17.956773  1383 raft_consensus.cc:2270] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:17.962160  1234 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:17.964850  1107 tablet_service.cc:1468] Processing CreateTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab (DEFAULT_TABLE table=default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.965240  1455 raft_consensus.cc:513] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.965930  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4e02676d24a4345908dbbc42fc9c1ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.970981   801 tablet_service.cc:1468] Processing CreateTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab (DEFAULT_TABLE table=default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.972036   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4e02676d24a4345908dbbc42fc9c1ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.974340   964 tablet_service.cc:1468] Processing CreateTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab (DEFAULT_TABLE table=default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:17.975538   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4e02676d24a4345908dbbc42fc9c1ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:17.976797  1510 tablet.cc:1722] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:17.902103  1457 raft_consensus.cc:513] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:17.923992   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7017192359304fa08eb3df34abc8c536"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:17.990159   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7017192359304fa08eb3df34abc8c536 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:17.998270   758 leader_election.cc:343] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:18.000392  1455 leader_election.cc:290] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.001974   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2df135033ee546f8bb9f4c4b54755b06" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:18.002110  1383 ts_tablet_manager.cc:1905] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.002523   985 raft_consensus.cc:3058] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.005435   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:18.013320  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:18.013996  1127 raft_consensus.cc:3058] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.020939   985 raft_consensus.cc:2466] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:17.955335  1507 tablet.cc:1722] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:18.045682  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:18.047490  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7dde9b060f4d839724e5f718519fb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:18.047562   962 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:18.048753   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7dde9b060f4d839724e5f718519fb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:18.050544  1230 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:18.050967  1230 ts_tablet_manager.cc:1397] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.298s	user 0.056s	sys 0.054s
I20250628 07:52:18.053012  1230 raft_consensus.cc:357] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.053640  1230 raft_consensus.cc:383] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.053910  1230 raft_consensus.cc:738] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:18.007673   757 leader_election.cc:343] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:18.054840  1229 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
W20250628 07:52:18.054683   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 47047.5us
I20250628 07:52:18.054494  1230 consensus_queue.cc:260] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.055226  1229 ts_tablet_manager.cc:1397] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.130s	user 0.010s	sys 0.003s
I20250628 07:52:18.055655   757 leader_election.cc:304] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:18.057461  1477 raft_consensus.cc:2802] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:18.057422  1505 tablet.cc:1722] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:18.058470  1477 raft_consensus.cc:695] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:18.059185  1477 consensus_queue.cc:237] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.063393   542 master_service.cc:948] ReplaceTablet: received request to replace tablet fa96605dbabe4f0fbc26b057232960a4 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:18.064481  1230 ts_tablet_manager.cc:1428] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20250628 07:52:18.019234  1397 raft_consensus.cc:491] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.066706  1397 raft_consensus.cc:513] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.044039   803 tablet_service.cc:1468] Processing CreateTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:18.069618   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f7dde9b060f4d839724e5f718519fb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:18.069322  1229 raft_consensus.cc:357] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.069968  1229 raft_consensus.cc:383] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.070186  1229 raft_consensus.cc:738] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.070835  1229 consensus_queue.cc:260] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.073001  1230 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:18.074733  1446 raft_consensus.cc:491] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.075131  1446 raft_consensus.cc:513] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.076771  1446 leader_election.cc:290] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:18.080184   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7a3e80af50b542ce94ea3cb2b2cef3b8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:18.081465   757 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:18.085175  1397 leader_election.cc:290] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.085600  1229 ts_tablet_manager.cc:1428] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.030s	user 0.005s	sys 0.001s
I20250628 07:52:18.086458  1229 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:18.088418  1230 tablet_bootstrap.cc:654] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.106319   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:18.107648   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:18.110378   758 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:18.110818   758 leader_election.cc:304] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:18.112205   758 consensus_peers.cc:489] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 474cec4459954e10a18de031946a2346. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:18.112911  1062 leader_election.cc:343] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:18.113610  1062 leader_election.cc:343] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:18.114037  1397 raft_consensus.cc:2747] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:18.117827  1062 leader_election.cc:304] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:18.123358  1234 tablet_bootstrap.cc:654] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.127673  1446 raft_consensus.cc:2747] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:18.135977  1229 tablet_bootstrap.cc:654] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.144798  1383 ts_tablet_manager.cc:1918] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.157069  1383 log.cc:1199] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:18.157833  1383 ts_tablet_manager.cc:1939] T 0ac2a2d045864749841a9b5de03d140c P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:18.161341   542 master_service.cc:948] ReplaceTablet: received request to replace tablet d62ecf32bcc54310b9114d5655be7673 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:18.163755  1127 raft_consensus.cc:2466] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:18.164438   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b2f11d5c89e84c33a307bcc83159841c" } modify_external_catalogs: true
I20250628 07:52:18.164916   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b2f11d5c89e84c33a307bcc83159841c" } modify_external_catalogs: true
I20250628 07:52:18.166141   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_48e99a2ce8f9404cb2bd12526a3f2f0e" } new_table_name: "default.Table_0adee4e1249840be952b0a99dba4c161" modify_external_catalogs: true
I20250628 07:52:18.123687  1457 leader_election.cc:290] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.139371  1397 raft_consensus.cc:491] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.167105  1397 raft_consensus.cc:513] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.169513  1397 leader_election.cc:290] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.139907  1455 consensus_queue.cc:1035] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:18.161270   758 consensus_peers.cc:489] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 84a8065c2d1349b792308ac3aec1a8fe. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:18.170534   758 leader_election.cc:304] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:18.172466  1455 raft_consensus.cc:2802] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
W20250628 07:52:18.174428   758 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:18.172904  1455 raft_consensus.cc:695] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:18.175725  1455 consensus_queue.cc:237] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.179859  1229 tablet_bootstrap.cc:492] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.180218  1229 ts_tablet_manager.cc:1397] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.094s	user 0.004s	sys 0.007s
I20250628 07:52:18.182662   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 0ac2a2d045864749841a9b5de03d140c (table default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]) successfully deleted
I20250628 07:52:18.182678   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa96605dbabe4f0fbc26b057232960a4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:18.183763   757 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:18.184152   757 leader_election.cc:304] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:18.185420  1383 tablet_replica.cc:331] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:18.186197  1383 raft_consensus.cc:2241] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:18.186632  1383 raft_consensus.cc:2270] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.189054  1477 raft_consensus.cc:2747] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:18.191799   985 raft_consensus.cc:3058] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.192749  1383 ts_tablet_manager.cc:1905] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.190490  1229 raft_consensus.cc:357] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.201766  1229 raft_consensus.cc:383] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.202131  1229 raft_consensus.cc:738] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.204165  1229 consensus_queue.cc:260] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.212253  1477 consensus_queue.cc:1035] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:18.213747  1229 ts_tablet_manager.cc:1428] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.033s	user 0.006s	sys 0.001s
I20250628 07:52:18.219599  1229 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:18.247378  1446 consensus_queue.cc:1035] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:18.250824  1230 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:18.251219  1230 ts_tablet_manager.cc:1397] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.178s	user 0.032s	sys 0.018s
I20250628 07:52:18.255916  1229 tablet_bootstrap.cc:654] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.268330  1244 ts_tablet_manager.cc:1918] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.268692  1244 log.cc:1199] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:18.269346  1244 ts_tablet_manager.cc:1939] T 2f7e2b290c69467aa42ca6328307bc3c P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:18.271870  1383 ts_tablet_manager.cc:1918] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.272214  1383 log.cc:1199] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:18.272830  1383 ts_tablet_manager.cc:1939] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:18.275218  1229 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.275565  1229 ts_tablet_manager.cc:1397] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.056s	user 0.005s	sys 0.003s
I20250628 07:52:18.281121  1234 tablet_bootstrap.cc:492] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:18.281495  1234 ts_tablet_manager.cc:1397] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.401s	user 0.009s	sys 0.000s
I20250628 07:52:18.289932  1229 raft_consensus.cc:357] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.290534  1229 raft_consensus.cc:383] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.290805  1229 raft_consensus.cc:738] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.299091  1402 ts_tablet_manager.cc:1918] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:18.299499  1402 log.cc:1199] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:18.301723  1402 ts_tablet_manager.cc:1939] T d39d14572c4647c7b4880c71a8707c08 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:18.302611  1229 consensus_queue.cc:260] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.305490  1446 consensus_queue.cc:1035] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:18.315454  1229 ts_tablet_manager.cc:1428] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.040s	user 0.006s	sys 0.000s
I20250628 07:52:18.317314   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (table default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]) successfully deleted
I20250628 07:52:18.318682   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_0979356e8b1a441fa942c0359545cfc9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:18.320341  1229 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:18.320864   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0979356e8b1a441fa942c0359545cfc9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:18.326536   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_09cc8edc019c4d26936ec96d72adbb0d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:18.327955   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_09cc8edc019c4d26936ec96d72adbb0d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:18.331178   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 because the tablet was not found. No further retry: Not found: Tablet not found: 3637ab35a2b74ce3a111c77cbec8b0b0
W20250628 07:52:18.332741   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2f7e2b290c69467aa42ca6328307bc3c because the tablet was not found. No further retry: Not found: Tablet not found: 2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:18.333580  1504 raft_consensus.cc:491] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.334008  1504 raft_consensus.cc:513] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.335600  1504 leader_election.cc:290] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.337137   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:18.338253  1383 tablet_replica.cc:331] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:18.339103  1383 raft_consensus.cc:2241] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:18.339813  1383 raft_consensus.cc:2270] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.344975   541 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:18.344988  1234 raft_consensus.cc:357] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.345582  1234 raft_consensus.cc:383] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.345803  1234 raft_consensus.cc:738] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.346246   541 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
I20250628 07:52:18.354890   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 2 got 0
I20250628 07:52:18.355436   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:18.356809   540 catalog_manager.cc:5582] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } }
I20250628 07:52:18.358258   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 1 got 0
I20250628 07:52:18.359344   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_a0e49295ed0e482eaa16a7761b01b32c [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 1 got 0
I20250628 07:52:18.359735  1229 tablet_bootstrap.cc:654] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.360186   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:18.360960   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:18.361392  1234 consensus_queue.cc:260] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.362145   824 raft_consensus.cc:2466] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:18.363605   920 leader_election.cc:304] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:18.364421  1504 raft_consensus.cc:2802] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:18.364714  1504 raft_consensus.cc:491] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:18.365003  1504 raft_consensus.cc:3058] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.373600  1502 raft_consensus.cc:491] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.374019  1502 raft_consensus.cc:513] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.375903  1504 raft_consensus.cc:513] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.378031   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:18.380369  1229 tablet_bootstrap.cc:492] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.380767  1504 leader_election.cc:290] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.375927  1502 leader_election.cc:290] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.380826  1229 ts_tablet_manager.cc:1397] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.061s	user 0.008s	sys 0.000s
I20250628 07:52:18.385921  1229 raft_consensus.cc:357] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.386687  1229 raft_consensus.cc:383] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.386991  1229 raft_consensus.cc:738] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.389210  1244 tablet_replica.cc:331] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:18.389911  1244 raft_consensus.cc:2241] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:18.390967  1244 raft_consensus.cc:2270] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.392602  1244 ts_tablet_manager.cc:1905] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.402189  1234 ts_tablet_manager.cc:1428] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.120s	user 0.006s	sys 0.000s
I20250628 07:52:18.403827  1229 consensus_queue.cc:260] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.411446  1229 ts_tablet_manager.cc:1428] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.028s	user 0.005s	sys 0.000s
I20250628 07:52:18.412851  1383 ts_tablet_manager.cc:1905] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.412973   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:18.413625   824 raft_consensus.cc:3058] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.425074  1234 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:18.427209  1229 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:18.433244  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 134480us
I20250628 07:52:18.433979  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:18.434691  1127 raft_consensus.cc:2466] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:18.435144  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83f8dcff296a4429a5f16bde59c8ebee" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:18.435925  1128 raft_consensus.cc:2391] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
I20250628 07:52:18.437685  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "872ebf0393fa40cf82feebaad1ca3fa3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:18.438144  1128 raft_consensus.cc:3058] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.444973  1128 raft_consensus.cc:2466] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:18.450964  1502 raft_consensus.cc:491] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.451440  1502 raft_consensus.cc:513] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.388502   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 2f7e2b290c69467aa42ca6328307bc3c (table default.Table_0ef22eef86e5432bb09a3f878ed94458 [id=c73657502c0a49f988579c354ca4bfbc]) successfully deleted
W20250628 07:52:18.458853   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 70386.8us
I20250628 07:52:18.407755   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:18.460467   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:18.461160   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:18.461243   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:18.464913   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:18.412403   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 34) took 1308 ms (client timeout 1000 ms). Trace:
I20250628 07:52:18.465554   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:18.469767  1230 raft_consensus.cc:357] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.470376  1230 raft_consensus.cc:383] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.470724  1230 raft_consensus.cc:738] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.470857  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:18.471424  1230 consensus_queue.cc:260] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.477737  1477 consensus_queue.cc:1035] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:18.485236  1504 raft_consensus.cc:491] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:18.485796  1504 raft_consensus.cc:513] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.487691   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
W20250628 07:52:18.488744   539 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 872ebf0393fa40cf82feebaad1ca3fa3 consensus state has no leader
I20250628 07:52:18.489501   539 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 148) took 1080 ms. Trace:
I20250628 07:52:18.492442  1230 ts_tablet_manager.cc:1428] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.241s	user 0.033s	sys 0.011s
I20250628 07:52:18.493470  1230 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:18.489915   539 rpcz_store.cc:276] 0628 07:52:17.409074 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.910660 (+501586us) service_pool.cc:225] Handling call
0628 07:52:17.911584 (+   924us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:17.931567 (+ 19983us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:17.932207 (+   640us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:17.932395 (+   188us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:18.337211 (+404816us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:18.489416 (+152205us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:18.258418 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:18.258901 (+   483us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:18.258936 (+    35us) write_op.cc:435] Acquired schema lock
0628 07:52:18.258956 (+    20us) tablet.cc:660] Decoding operations
0628 07:52:18.259261 (+   305us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:18.259309 (+    48us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:18.259342 (+    33us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:18.259512 (+   170us) tablet.cc:699] Row locks acquired
0628 07:52:18.259531 (+    19us) write_op.cc:260] PREPARE: finished
0628 07:52:18.259663 (+   132us) write_op.cc:270] Start()
0628 07:52:18.259748 (+    85us) write_op.cc:276] Timestamp: P: 1751097138259581 usec, L: 0
0628 07:52:18.259770 (+    22us) op_driver.cc:348] REPLICATION: starting
0628 07:52:18.260302 (+   532us) log.cc:844] Serialized 405 byte log entry
0628 07:52:18.330893 (+ 70591us) op_driver.cc:464] REPLICATION: finished
0628 07:52:18.332421 (+  1528us) write_op.cc:301] APPLY: starting
0628 07:52:18.332463 (+    42us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:18.332532 (+    69us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:18.332618 (+    86us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:18.332754 (+   136us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:18.332863 (+   109us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:18.332903 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:18.333037 (+   134us) log.cc:844] Serialized 23 byte log entry
0628 07:52:18.333209 (+   172us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:18.333277 (+    68us) write_op.cc:454] Released schema lock
0628 07:52:18.333311 (+    34us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:18.389492 (+ 56181us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:18.389517 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1419,"apply.run_cpu_time_us":1072,"apply.run_wall_time_us":1075,"num_ops":1,"prepare.queue_time_us":324899,"prepare.run_cpu_time_us":2260,"prepare.run_wall_time_us":130958,"raft.queue_time_us":4367,"raft.run_cpu_time_us":196,"raft.run_wall_time_us":197,"replication_time_us":71039}]]}
I20250628 07:52:18.500967   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.502009   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.503185   962 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.504174   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.527339   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet d39d14572c4647c7b4880c71a8707c08 (table default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]) successfully deleted
I20250628 07:52:18.528468  1402 tablet_replica.cc:331] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:18.529685  1402 raft_consensus.cc:2241] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:18.530653  1526 log.cc:927] Time spent T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Append to log took a long time: real 0.081s	user 0.001s	sys 0.000s
W20250628 07:52:18.465541   546 rpcz_store.cc:269] 0628 07:52:17.104336 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.153740 (+ 49404us) service_pool.cc:225] Handling call
0628 07:52:17.160204 (+  6464us) spinlock_profiling.cc:243] Waited 3.52 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:18.412321 (+1252117us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:18.191046 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:18.191470 (+   424us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:18.191491 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:18.191504 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:18.191757 (+   253us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:18.191794 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:18.191826 (+    32us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:18.191972 (+   146us) tablet.cc:699] Row locks acquired
0628 07:52:18.191982 (+    10us) write_op.cc:260] PREPARE: finished
0628 07:52:18.212970 (+ 20988us) write_op.cc:270] Start()
0628 07:52:18.213056 (+    86us) write_op.cc:276] Timestamp: P: 1751097138192000 usec, L: 0
0628 07:52:18.213072 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:18.241283 (+ 28211us) log.cc:844] Serialized 533 byte log entry
0628 07:52:18.258153 (+ 16870us) spinlock_profiling.cc:243] Waited 27.6 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:18.330670 (+ 72517us) op_driver.cc:464] REPLICATION: finished
0628 07:52:18.331088 (+   418us) write_op.cc:301] APPLY: starting
0628 07:52:18.331138 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:18.331225 (+    87us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:18.331244 (+    19us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:18.331378 (+   134us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:18.331509 (+   131us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:18.331549 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:18.331725 (+   176us) log.cc:844] Serialized 23 byte log entry
0628 07:52:18.331903 (+   178us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:18.331958 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:18.331989 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:18.332205 (+   216us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:18.332228 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":13,"spinlock_wait_cycles":8090368,"child_traces":[["op",{"apply.queue_time_us":284,"apply.run_cpu_time_us":1092,"apply.run_wall_time_us":1092,"num_ops":1,"prepare.queue_time_us":260934,"prepare.run_cpu_time_us":2743,"prepare.run_wall_time_us":67237,"replication_time_us":117468,"spinlock_wait_cycles":63482496}]]}
I20250628 07:52:18.531983  1230 tablet_bootstrap.cc:654] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.549853   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.530162  1402 raft_consensus.cc:2270] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.553805   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:18.555505   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:18.579720   758 consensus_peers.cc:489] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:18.584352  1230 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:18.584802  1230 ts_tablet_manager.cc:1397] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.092s	user 0.007s	sys 0.004s
I20250628 07:52:18.552562  1402 ts_tablet_manager.cc:1905] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.587626  1230 raft_consensus.cc:357] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.588402  1230 raft_consensus.cc:383] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.588694  1230 raft_consensus.cc:738] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.591248   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2b51d6076e344c8f8e388b77b4b23928" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:18.592495   920 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:18.593277   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 147251us
I20250628 07:52:18.593549   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.594085   921 leader_election.cc:304] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:18.595736  1504 leader_election.cc:290] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.596248  1504 raft_consensus.cc:3058] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.596807   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa96605dbabe4f0fbc26b057232960a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:18.597368   825 raft_consensus.cc:2466] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:18.598654   920 leader_election.cc:304] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:18.599824   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:18.601994  1229 tablet_bootstrap.cc:654] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.606057  1230 consensus_queue.cc:260] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.608557  1502 leader_election.cc:290] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.616879  1459 consensus_queue.cc:1035] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:18.622674   544 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 149) took 1179 ms. Trace:
I20250628 07:52:18.626111  1230 ts_tablet_manager.cc:1428] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.041s	user 0.008s	sys 0.000s
I20250628 07:52:18.626883  1230 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:18.622982   544 rpcz_store.cc:276] 0628 07:52:17.443343 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.923626 (+480283us) service_pool.cc:225] Handling call
0628 07:52:17.990720 (+ 67094us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:17.992824 (+  2104us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:18.621998 (+629174us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:18.622318 (+   320us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:18.622624 (+   306us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:18.550351 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:18.550769 (+   418us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:18.550804 (+    35us) write_op.cc:435] Acquired schema lock
0628 07:52:18.550816 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:18.551200 (+   384us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:18.551250 (+    50us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:18.551270 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:18.551440 (+   170us) tablet.cc:699] Row locks acquired
0628 07:52:18.551452 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:18.551552 (+   100us) write_op.cc:270] Start()
0628 07:52:18.551629 (+    77us) write_op.cc:276] Timestamp: P: 1751097138551490 usec, L: 0
0628 07:52:18.551644 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:18.610092 (+ 58448us) log.cc:844] Serialized 433 byte log entry
0628 07:52:18.611633 (+  1541us) spinlock_profiling.cc:243] Waited 57.8 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:18.618424 (+  6791us) op_driver.cc:464] REPLICATION: finished
0628 07:52:18.618674 (+   250us) write_op.cc:301] APPLY: starting
0628 07:52:18.618727 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:18.618812 (+    85us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:18.618829 (+    17us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:18.619211 (+   382us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:18.619339 (+   128us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:18.619378 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:18.619526 (+   148us) log.cc:844] Serialized 29 byte log entry
0628 07:52:18.619696 (+   170us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:18.619760 (+    64us) write_op.cc:454] Released schema lock
0628 07:52:18.619791 (+    31us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":79,"apply.run_cpu_time_us":1291,"apply.run_wall_time_us":1292,"num_ops":2,"prepare.queue_time_us":466619,"prepare.run_cpu_time_us":2644,"prepare.run_wall_time_us":80481,"raft.queue_time_us":91,"raft.run_cpu_time_us":589,"raft.run_wall_time_us":590,"replication_time_us":66695,"spinlock_wait_cycles":132884352}]]}
I20250628 07:52:18.647781  1229 tablet_bootstrap.cc:492] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.648239  1229 ts_tablet_manager.cc:1397] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.221s	user 0.018s	sys 0.023s
I20250628 07:52:18.650450  1229 raft_consensus.cc:357] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.651074  1229 raft_consensus.cc:383] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.651304  1229 raft_consensus.cc:738] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.651811  1229 consensus_queue.cc:260] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.667395  1229 ts_tablet_manager.cc:1428] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.019s	user 0.007s	sys 0.000s
I20250628 07:52:18.667325   541 master_service.cc:948] ReplaceTablet: received request to replace tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:18.669240  1229 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:18.645059  1230 tablet_bootstrap.cc:654] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:18.676169   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20250628 07:52:18.677853  1229 tablet_bootstrap.cc:654] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.635900  1536 raft_consensus.cc:2802] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:18.693691  1234 tablet_bootstrap.cc:654] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.696830   546 master_service.cc:948] ReplaceTablet: received request to replace tablet 474cec4459954e10a18de031946a2346 from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:18.699072   757 consensus_peers.cc:489] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:18.701649  1536 raft_consensus.cc:491] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:18.702003  1536 raft_consensus.cc:3058] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.703632  1402 ts_tablet_manager.cc:1918] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:18.636644   921 leader_election.cc:304] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
W20250628 07:52:18.705354   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 68780.5us
I20250628 07:52:18.705431  1539 raft_consensus.cc:2802] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:18.705894  1539 raft_consensus.cc:695] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:18.706267  1402 log.cc:1199] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:18.706630  1539 consensus_queue.cc:237] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.707778  1402 ts_tablet_manager.cc:1939] T c84f2570b2da45d0b3ce6428723263a5 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:18.714831   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c0546593fb7742e3b88d4b60b53ca658" } new_table_name: "default.Table_5a2877b9c29b49c18d0909b99b08fa80" modify_external_catalogs: true
W20250628 07:52:18.715369   921 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4
W20250628 07:52:18.716123   921 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:18.716454   921 leader_election.cc:304] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:18.709882  1230 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:18.718691   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5f59babfe3234a8e83bfab516b219032" } modify_external_catalogs: true
I20250628 07:52:18.718971  1230 ts_tablet_manager.cc:1397] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.092s	user 0.012s	sys 0.000s
I20250628 07:52:18.719141   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5f59babfe3234a8e83bfab516b219032" } modify_external_catalogs: true
I20250628 07:52:18.721789  1230 raft_consensus.cc:357] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.722432  1230 raft_consensus.cc:383] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.722673  1230 raft_consensus.cc:738] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.725507   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet c84f2570b2da45d0b3ce6428723263a5 (table default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]) successfully deleted
I20250628 07:52:18.726234  1230 consensus_queue.cc:260] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.727563   824 raft_consensus.cc:2466] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:18.727651   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 84a8065c2d1349b792308ac3aec1a8fe because the tablet was not found. No further retry: Not found: Tablet not found: 84a8065c2d1349b792308ac3aec1a8fe
W20250628 07:52:18.735448   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 16a8bee148664e948529d32c0cf2c793 consensus state has no leader
W20250628 07:52:18.738199   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba already in progress: creating tablet
I20250628 07:52:18.750346  1244 ts_tablet_manager.cc:1918] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.717444  1537 raft_consensus.cc:2747] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:18.753655  1230 ts_tablet_manager.cc:1428] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.034s	user 0.006s	sys 0.000s
I20250628 07:52:18.762282  1230 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:18.764503  1234 tablet_bootstrap.cc:492] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:18.764856  1234 ts_tablet_manager.cc:1397] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.340s	user 0.008s	sys 0.000s
I20250628 07:52:18.765326  1244 log.cc:1199] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/84a8065c2d1349b792308ac3aec1a8fe
I20250628 07:52:18.766048  1244 ts_tablet_manager.cc:1939] T 84a8065c2d1349b792308ac3aec1a8fe P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:18.769429  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 83) took 1086 ms. Trace:
I20250628 07:52:18.769685  1229 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.770069  1229 ts_tablet_manager.cc:1397] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.101s	user 0.018s	sys 0.001s
I20250628 07:52:18.769727  1244 rpcz_store.cc:276] 0628 07:52:17.683217 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.847155 (+163938us) service_pool.cc:225] Handling call
0628 07:52:18.389043 (+541888us) ts_tablet_manager.cc:1145] Deleting tablet 84a8065c2d1349b792308ac3aec1a8fe
0628 07:52:18.389078 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:18.516630 (+127552us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:18.750141 (+233511us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:18.769370 (+ 19229us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":541369}
I20250628 07:52:18.774418  1383 ts_tablet_manager.cc:1918] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.774744  1383 log.cc:1199] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/deb5a9ae7ab5463cae35ff1086fd8348
I20250628 07:52:18.775381  1383 ts_tablet_manager.cc:1939] T deb5a9ae7ab5463cae35ff1086fd8348 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:18.775379  1234 raft_consensus.cc:357] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.775969  1234 raft_consensus.cc:383] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.776186  1234 raft_consensus.cc:738] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.786511  1536 raft_consensus.cc:513] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.789444  1244 tablet_replica.cc:331] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:18.790228  1244 raft_consensus.cc:2241] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:18.789985  1229 raft_consensus.cc:357] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.790658  1229 raft_consensus.cc:383] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.790725  1244 raft_consensus.cc:2270] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.790922  1229 raft_consensus.cc:738] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.792485  1536 leader_election.cc:290] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:18.793267   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa96605dbabe4f0fbc26b057232960a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:18.793783   824 raft_consensus.cc:3058] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:18.776700  1234 consensus_queue.cc:260] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.795755  1504 raft_consensus.cc:2747] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:18.797264  1234 ts_tablet_manager.cc:1428] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.007s	sys 0.001s
I20250628 07:52:18.797327  1244 ts_tablet_manager.cc:1905] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.798844  1234 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:18.806813  1234 tablet_bootstrap.cc:654] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.791541  1229 consensus_queue.cc:260] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.820698  1229 ts_tablet_manager.cc:1428] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.050s	user 0.006s	sys 0.000s
I20250628 07:52:18.833626  1229 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:18.845526  1383 tablet_replica.cc:331] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:18.846197  1383 raft_consensus.cc:2241] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:18.846655  1383 raft_consensus.cc:2270] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.847878  1229 tablet_bootstrap.cc:654] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.851451  1383 ts_tablet_manager.cc:1905] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.854602   824 raft_consensus.cc:2466] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:18.855751   920 leader_election.cc:304] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:18.856420  1504 raft_consensus.cc:2802] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:18.856932  1504 raft_consensus.cc:695] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:18.857756  1504 consensus_queue.cc:237] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:18.870352   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 872ebf0393fa40cf82feebaad1ca3fa3 consensus state has no leader
I20250628 07:52:18.871626   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet deb5a9ae7ab5463cae35ff1086fd8348 (table default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]) successfully deleted
W20250628 07:52:18.872478   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 84a8065c2d1349b792308ac3aec1a8fe because the tablet was not found. No further retry: Not found: Tablet not found: 84a8065c2d1349b792308ac3aec1a8fe
W20250628 07:52:18.873427   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba already in progress: creating tablet
W20250628 07:52:18.874341   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2f7e2b290c69467aa42ca6328307bc3c because the tablet was not found. No further retry: Not found: Tablet not found: 2f7e2b290c69467aa42ca6328307bc3c
I20250628 07:52:18.874547  1244 ts_tablet_manager.cc:1918] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.875026  1244 log.cc:1199] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/0ac2a2d045864749841a9b5de03d140c
W20250628 07:52:18.875414   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0ac2a2d045864749841a9b5de03d140c because the tablet was not found. No further retry: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:18.875902  1244 ts_tablet_manager.cc:1939] T 0ac2a2d045864749841a9b5de03d140c P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
W20250628 07:52:18.876276   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 because the tablet was not found. No further retry: Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:18.877105   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 84a8065c2d1349b792308ac3aec1a8fe (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
I20250628 07:52:18.880311  1229 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:18.880728  1229 ts_tablet_manager.cc:1397] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.059s	user 0.009s	sys 0.000s
I20250628 07:52:18.894305  1244 tablet_replica.cc:331] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:18.895339   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 0ac2a2d045864749841a9b5de03d140c (table default.Table_181d43b0aeef45e2bc96b51b8392b136 [id=67f757e6b2a6484cb069ddf5fc92026b]) successfully deleted
W20250628 07:52:18.896191   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 3637ab35a2b74ce3a111c77cbec8b0b0 because the tablet was not found. No further retry: Not found: Tablet not found: 3637ab35a2b74ce3a111c77cbec8b0b0
I20250628 07:52:18.897217  1229 raft_consensus.cc:357] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.897845  1229 raft_consensus.cc:383] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:18.903139  1229 raft_consensus.cc:738] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:18.903733  1229 consensus_queue.cc:260] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:18.907235  1244 raft_consensus.cc:2241] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:18.908151  1244 raft_consensus.cc:2270] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.915746  1244 ts_tablet_manager.cc:1905] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.918563  1229 ts_tablet_manager.cc:1428] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.038s	user 0.007s	sys 0.000s
I20250628 07:52:18.920558  1229 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:18.938125  1383 ts_tablet_manager.cc:1918] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.938503  1383 log.cc:1199] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:18.941045  1383 ts_tablet_manager.cc:1939] T 67a81eb4342740e5bc16ee7f5eacb489 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:18.945215  1229 tablet_bootstrap.cc:654] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:18.951982  1383 tablet_replica.cc:331] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:18.952739  1383 raft_consensus.cc:2241] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:18.953156  1383 raft_consensus.cc:2270] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.954808  1383 ts_tablet_manager.cc:1905] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:18.910728   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e1df74fb3dfa455bb3300ad20e7a2268"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:18.974220   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e1df74fb3dfa455bb3300ad20e7a2268 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:18.974642  1244 ts_tablet_manager.cc:1918] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:18.974980  1244 log.cc:1199] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/e71a4b8cfa3a41d1837ca6fd2e2a3e67
I20250628 07:52:18.975647  1244 ts_tablet_manager.cc:1939] T e71a4b8cfa3a41d1837ca6fd2e2a3e67 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
W20250628 07:52:18.978035   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 81854.3us
I20250628 07:52:18.978788   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 67a81eb4342740e5bc16ee7f5eacb489 (table default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]) successfully deleted
I20250628 07:52:18.981607  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 90) took 1074 ms. Trace:
I20250628 07:52:18.982010  1244 rpcz_store.cc:276] 0628 07:52:17.906686 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.906793 (+   107us) service_pool.cc:225] Handling call
0628 07:52:18.894129 (+987336us) ts_tablet_manager.cc:1145] Deleting tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67
0628 07:52:18.894172 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:18.942959 (+ 48787us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:18.974414 (+ 31455us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:18.981521 (+  7107us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":986936}
I20250628 07:52:18.983497   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 (table default.Table_801ed4c53ecc49cb9a54dd6547a8da6d [id=fe8757a1a0404748a03fe85427c65c28]) successfully deleted
I20250628 07:52:18.986158   543 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet a05bd99137cb49b58f7dd21bcd0d3234
I20250628 07:52:18.994247  1244 tablet_replica.cc:331] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:18.994939  1244 raft_consensus.cc:2241] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:18.995312  1244 raft_consensus.cc:2270] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:18.998772  1244 ts_tablet_manager.cc:1905] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.026284  1383 ts_tablet_manager.cc:1918] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:19.026643  1383 log.cc:1199] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:19.027287  1383 ts_tablet_manager.cc:1939] T 2df135033ee546f8bb9f4c4b54755b06 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:19.038971   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 2df135033ee546f8bb9f4c4b54755b06 (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:19.041229  1383 tablet_replica.cc:331] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:19.041908  1383 raft_consensus.cc:2241] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.042328  1383 raft_consensus.cc:2270] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.053488  1230 tablet_bootstrap.cc:654] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.057833  1383 ts_tablet_manager.cc:1905] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.066149   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_85234bd18f8e42a3b62dc723411c065a" } new_table_name: "default.Table_60e2e342cb184e3da1a5b16ed3218f94" modify_external_catalogs: true
I20250628 07:52:19.067670  1558 tablet.cc:1722] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:19.070361  1557 tablet.cc:1722] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:19.096357  1244 ts_tablet_manager.cc:1918] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.096709  1244 log.cc:1199] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/deb5a9ae7ab5463cae35ff1086fd8348
I20250628 07:52:19.109916  1244 ts_tablet_manager.cc:1939] T deb5a9ae7ab5463cae35ff1086fd8348 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.118078  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 91) took 1210 ms. Trace:
I20250628 07:52:19.118415  1244 rpcz_store.cc:276] 0628 07:52:17.908041 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.908149 (+   108us) service_pool.cc:225] Handling call
0628 07:52:18.994049 (+1085900us) ts_tablet_manager.cc:1145] Deleting tablet deb5a9ae7ab5463cae35ff1086fd8348
0628 07:52:18.994102 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:19.026608 (+ 32506us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:19.096139 (+ 69531us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:19.118016 (+ 21877us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1085474}
I20250628 07:52:19.119874  1244 tablet_replica.cc:331] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.120045   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet deb5a9ae7ab5463cae35ff1086fd8348 (table default.Table_712625afc1034446a1f0bfd3618ea308 [id=abe4c5ddbdbd45d78a6a8dbaa360c2cd]) successfully deleted
I20250628 07:52:19.120693  1244 raft_consensus.cc:2241] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:19.121631  1244 raft_consensus.cc:2270] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:19.122913   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet b2d49e5a06384758bce5e7328b6a10f8 has no consensus state
I20250628 07:52:19.123569  1244 ts_tablet_manager.cc:1905] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.136569   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:19.138069   541 catalog_manager.cc:6652] ReplaceTablet: tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 deleted and replaced by tablet 8feee3be1a664cae9da417b190163a94
I20250628 07:52:19.143169   801 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:19.145429   962 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:19.151360  1446 raft_consensus.cc:491] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.152119  1446 raft_consensus.cc:513] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.165684  1244 ts_tablet_manager.cc:1918] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.166110  1244 log.cc:1199] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/0f22700fc70d4f14ae51a834900db22b
I20250628 07:52:19.166170  1383 ts_tablet_manager.cc:1918] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.166525  1383 log.cc:1199] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/0f22700fc70d4f14ae51a834900db22b
I20250628 07:52:19.166880  1244 ts_tablet_manager.cc:1939] T 0f22700fc70d4f14ae51a834900db22b P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.167235  1383 ts_tablet_manager.cc:1939] T 0f22700fc70d4f14ae51a834900db22b P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:19.167696  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 92) took 1225 ms. Trace:
I20250628 07:52:19.168043  1244 rpcz_store.cc:276] 0628 07:52:17.942488 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.942596 (+   108us) service_pool.cc:225] Handling call
0628 07:52:19.119717 (+1177121us) ts_tablet_manager.cc:1145] Deleting tablet 0f22700fc70d4f14ae51a834900db22b
0628 07:52:19.119762 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:19.142816 (+ 23054us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:19.165418 (+ 22602us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:19.167636 (+  2218us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1176652}
I20250628 07:52:19.169597  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 87) took 1267 ms. Trace:
I20250628 07:52:19.169852  1244 rpcz_store.cc:276] 0628 07:52:17.902616 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.943964 (+ 41348us) service_pool.cc:225] Handling call
0628 07:52:19.169327 (+1225363us) ts_tablet_manager.cc:1145] Deleting tablet ef449bd95fc64bf28c854b03a219c7b4
0628 07:52:19.169374 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:19.169544 (+   170us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1224802}
I20250628 07:52:19.170974  1244 tablet_replica.cc:331] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.171608  1244 raft_consensus.cc:2241] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:19.172384  1244 raft_consensus.cc:2270] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.174268  1244 ts_tablet_manager.cc:1905] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.178851  1528 raft_consensus.cc:491] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.179325  1528 raft_consensus.cc:513] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.182010  1383 tablet_replica.cc:331] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:19.183718  1383 raft_consensus.cc:2241] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.184152  1383 raft_consensus.cc:2270] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:19.186375   921 leader_election.cc:343] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:19.187067  1383 ts_tablet_manager.cc:1905] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.181071  1528 leader_election.cc:290] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:19.194063  1446 leader_election.cc:290] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:19.205600  1244 ts_tablet_manager.cc:1918] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.206005  1244 log.cc:1199] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/474cec4459954e10a18de031946a2346
I20250628 07:52:19.206789  1244 ts_tablet_manager.cc:1939] T 474cec4459954e10a18de031946a2346 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.210383  1244 tablet_replica.cc:331] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.211331  1244 raft_consensus.cc:2241] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:19.212165  1244 raft_consensus.cc:2270] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.225898  1244 ts_tablet_manager.cc:1905] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:19.226258  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 55755.8us
I20250628 07:52:19.229979   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:19.230927  1383 ts_tablet_manager.cc:1918] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.231288  1383 log.cc:1199] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/474cec4459954e10a18de031946a2346
I20250628 07:52:19.231225   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d62ecf32bcc54310b9114d5655be7673" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:19.231993  1383 ts_tablet_manager.cc:1939] T 474cec4459954e10a18de031946a2346 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:19.233901   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d62ecf32bcc54310b9114d5655be7673" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.232950   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.241571  1556 raft_consensus.cc:491] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.246114  1383 tablet_replica.cc:331] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:19.246095  1556 raft_consensus.cc:513] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.247756  1383 raft_consensus.cc:2241] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:19.248535  1383 raft_consensus.cc:2270] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.249212  1457 consensus_queue.cc:1035] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:19.250144  1244 ts_tablet_manager.cc:1918] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.250255  1383 ts_tablet_manager.cc:1905] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.250538  1244 log.cc:1199] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:19.256223  1244 ts_tablet_manager.cc:1939] T 67a81eb4342740e5bc16ee7f5eacb489 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.257504  1244 tablet_replica.cc:331] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.258149  1244 raft_consensus.cc:2241] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:19.258731  1244 pending_rounds.cc:70] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Trying to abort 1 pending ops.
I20250628 07:52:19.258981  1244 pending_rounds.cc:77] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493877515046912 op_type: NO_OP noop_request { }
I20250628 07:52:19.259330  1244 raft_consensus.cc:2887] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
W20250628 07:52:19.249200  1062 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.257531  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:19.259624  1244 raft_consensus.cc:2270] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.261350  1244 ts_tablet_manager.cc:1905] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.263051  1457 raft_consensus.cc:491] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.263468  1457 raft_consensus.cc:513] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.265436  1457 leader_election.cc:290] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.266165   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2b51d6076e344c8f8e388b77b4b23928" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:19.266733   984 raft_consensus.cc:2466] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:19.267879   757 leader_election.cc:304] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:19.268591  1457 raft_consensus.cc:2802] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:19.268935  1457 raft_consensus.cc:491] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:19.269241  1457 raft_consensus.cc:3058] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.274169  1562 tablet_replica.cc:331] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:19.274895  1062 leader_election.cc:343] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.275619  1457 raft_consensus.cc:513] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.277968   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2b51d6076e344c8f8e388b77b4b23928" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:19.278468   984 raft_consensus.cc:3058] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:19.278465  1062 leader_election.cc:343] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.275219  1562 raft_consensus.cc:2241] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.279532  1562 raft_consensus.cc:2270] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.279094  1062 leader_election.cc:304] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:19.281530  1527 raft_consensus.cc:2747] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:19.282887  1062 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.285919   984 raft_consensus.cc:2466] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:19.286911   757 leader_election.cc:304] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:19.287680  1477 raft_consensus.cc:2802] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:19.292778   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_78d794b2cff54bdf88d1996d140e4de8" } modify_external_catalogs: true
I20250628 07:52:19.293480   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_78d794b2cff54bdf88d1996d140e4de8" } modify_external_catalogs: true
I20250628 07:52:19.301839  1244 ts_tablet_manager.cc:1918] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:19.302261  1244 log.cc:1199] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:19.303045  1244 ts_tablet_manager.cc:1939] T 2df135033ee546f8bb9f4c4b54755b06 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.305301  1244 tablet_replica.cc:331] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.306178  1244 raft_consensus.cc:2241] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.306747  1244 raft_consensus.cc:2270] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.310717  1229 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:19.311102  1229 ts_tablet_manager.cc:1397] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.391s	user 0.036s	sys 0.041s
I20250628 07:52:19.308955  1244 ts_tablet_manager.cc:1905] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.315737  1229 raft_consensus.cc:357] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.316798  1229 raft_consensus.cc:383] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.317050   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 63348a64bb7044dbad9065afff179fbc
I20250628 07:52:19.317055  1229 raft_consensus.cc:738] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.321523  1062 leader_election.cc:304] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:19.322325  1527 raft_consensus.cc:2747] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:19.320581  1229 consensus_queue.cc:260] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.324201  1229 ts_tablet_manager.cc:1428] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:19.329633  1504 raft_consensus.cc:491] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.330101  1504 raft_consensus.cc:513] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.331789  1504 leader_election.cc:290] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.345187  1229 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:19.350414  1229 tablet_bootstrap.cc:654] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.353845   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:19.355507   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:19.362843  1383 ts_tablet_manager.cc:1918] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:19.363181  1383 log.cc:1199] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:19.363849  1383 ts_tablet_manager.cc:1939] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:19.364506   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_fd173dac22494e51a7adca1ec4817a45"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:19.366950   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_fd173dac22494e51a7adca1ec4817a45 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:19.370710  1244 ts_tablet_manager.cc:1918] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:19.371065  1244 log.cc:1199] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c4a5d0deb0a049a2b6cc2ac5a8dd3c17
W20250628 07:52:19.363018  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 79969.1us
I20250628 07:52:19.371773  1244 ts_tablet_manager.cc:1939] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.373142   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:19.371559  1457 leader_election.cc:290] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.378494   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.379053   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7a3e80af50b542ce94ea3cb2b2cef3b8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.379627   824 raft_consensus.cc:2466] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:19.380662   920 leader_election.cc:304] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
W20250628 07:52:19.381162   921 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:19.381448  1477 raft_consensus.cc:695] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:19.382093  1504 raft_consensus.cc:2802] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:19.382448  1504 raft_consensus.cc:491] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:19.382766  1504 raft_consensus.cc:3058] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.382212  1477 consensus_queue.cc:237] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.410549  1504 raft_consensus.cc:513] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.413838   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7a3e80af50b542ce94ea3cb2b2cef3b8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:19.414366   824 raft_consensus.cc:3058] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:19.414845   921 leader_election.cc:343] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:19.415839  1536 raft_consensus.cc:491] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.416249  1536 raft_consensus.cc:513] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.417152  1230 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:19.417609  1230 ts_tablet_manager.cc:1397] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.656s	user 0.012s	sys 0.008s
I20250628 07:52:19.418006  1536 leader_election.cc:290] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.419394   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.419672   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 0f22700fc70d4f14ae51a834900db22b (table default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]) successfully deleted
I20250628 07:52:19.419800  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:19.420392   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 474cec4459954e10a18de031946a2346 (table default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]) successfully deleted
I20250628 07:52:19.420329  1128 raft_consensus.cc:2466] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:19.420614   920 leader_election.cc:343] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.420883   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]) successfully deleted
I20250628 07:52:19.421252   921 leader_election.cc:304] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:19.421408   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 0f22700fc70d4f14ae51a834900db22b (table default.Table_1c78cba777ca43b6b253acb1d9c35a3a [id=75f22c573f5c4733a461d55803aca62b]) successfully deleted
W20250628 07:52:19.421936   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet ef449bd95fc64bf28c854b03a219c7b4 because the tablet was not found. No further retry: Not found: Tablet not found: ef449bd95fc64bf28c854b03a219c7b4
I20250628 07:52:19.422161  1536 raft_consensus.cc:2802] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:19.422447  1536 raft_consensus.cc:491] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:19.422466   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 474cec4459954e10a18de031946a2346 (table default.Table_344e153bf04b4bf19e388f9f092ce6fb [id=960298e525be4269933555bb6c2ff7b5]) successfully deleted
I20250628 07:52:19.422724  1536 raft_consensus.cc:3058] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.290282  1556 leader_election.cc:290] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:19.294253  1562 ts_tablet_manager.cc:1905] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.422925   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 67a81eb4342740e5bc16ee7f5eacb489 (table default.Table_682dd2d7af414997aa391924c47d7b31 [id=454c89186e5f4af69be78e44ecfa15c8]) successfully deleted
I20250628 07:52:19.424383   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 2df135033ee546f8bb9f4c4b54755b06 (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:19.319525  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 63348a64bb7044dbad9065afff179fbc with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:19 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:19.428813  1536 raft_consensus.cc:513] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.430917  1536 leader_election.cc:290] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.431772   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:19.433065  1229 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:19.433437  1229 ts_tablet_manager.cc:1397] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.088s	user 0.011s	sys 0.006s
I20250628 07:52:19.433212  1457 consensus_queue.cc:1035] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:19.435353   920 leader_election.cc:343] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.435523  1229 raft_consensus.cc:357] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.436072  1229 raft_consensus.cc:383] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.436316  1229 raft_consensus.cc:738] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.438045   824 raft_consensus.cc:2466] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:19.440779   920 leader_election.cc:304] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:19.442203  1536 raft_consensus.cc:2802] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:19.444564  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:19.445226  1128 raft_consensus.cc:3058] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.453466  1229 consensus_queue.cc:260] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.412887  1504 leader_election.cc:290] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.472952  1229 ts_tablet_manager.cc:1428] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.039s	user 0.005s	sys 0.001s
I20250628 07:52:19.484683  1128 raft_consensus.cc:2466] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:19.486182  1229 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:19.486488   758 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928
W20250628 07:52:19.487823   758 leader_election.cc:343] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928
W20250628 07:52:19.488896  1234 thread.cc:640] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.680s	user 0.069s	sys 0.136s
W20250628 07:52:19.489791  1234 thread.cc:606] log-alloc [worker] (thread pool) Time spent starting thread: real 0.681s	user 0.069s	sys 0.136s
I20250628 07:52:19.493892  1229 tablet_bootstrap.cc:654] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:19.512555   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba already in progress: creating tablet
I20250628 07:52:19.508983  1536 raft_consensus.cc:695] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:19.513989  1536 consensus_queue.cc:237] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.515920   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]) successfully deleted
I20250628 07:52:19.518314   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 63348a64bb7044dbad9065afff179fbc with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:19 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:19.519110  1244 tablet_replica.cc:331] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:19.519858  1244 raft_consensus.cc:2241] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.520390  1244 raft_consensus.cc:2270] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.532055  1244 ts_tablet_manager.cc:1905] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.540331  1229 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:19.540731  1229 ts_tablet_manager.cc:1397] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.055s	user 0.008s	sys 0.000s
I20250628 07:52:19.541630  1569 raft_consensus.cc:491] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.541738   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 63348a64bb7044dbad9065afff179fbc with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:19 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:19.542361  1383 tablet_replica.cc:331] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:19.542037  1569 raft_consensus.cc:513] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.543037  1383 raft_consensus.cc:2241] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:19.543535  1383 raft_consensus.cc:2270] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:19.543833  1569 leader_election.cc:290] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.546211  1383 ts_tablet_manager.cc:1905] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.546438   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d39d14572c4647c7b4880c71a8707c08" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:19.548167   920 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.557904   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_055d241428174f7891615d2422293f39"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:19.560192   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a70f9c604ca44b7eae2b205bc80fdf1d" } new_table_name: "default.Table_6b7424c1f64e4d9a90a106ee7637dac0" modify_external_catalogs: true
I20250628 07:52:19.568511  1229 raft_consensus.cc:357] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.569226  1229 raft_consensus.cc:383] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.575462  1229 raft_consensus.cc:738] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.577311  1536 raft_consensus.cc:491] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.577955  1536 raft_consensus.cc:513] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.582564   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.576382  1229 consensus_queue.cc:260] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:19.587970   920 leader_election.cc:343] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.588905  1536 leader_election.cc:290] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:19.588371   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_07319c0694ca4307b243e8938d21939b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:19.591594  1383 ts_tablet_manager.cc:1918] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:19.591934  1383 log.cc:1199] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/63348a64bb7044dbad9065afff179fbc
I20250628 07:52:19.592558  1383 ts_tablet_manager.cc:1939] T 63348a64bb7044dbad9065afff179fbc P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:19.612824  1234 tablet_bootstrap.cc:492] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:19.609776  1229 ts_tablet_manager.cc:1428] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.069s	user 0.006s	sys 0.003s
I20250628 07:52:19.613421  1234 ts_tablet_manager.cc:1397] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.815s	user 0.074s	sys 0.140s
I20250628 07:52:19.614856   541 master_service.cc:948] ReplaceTablet: received request to replace tablet 3223005021ae4e38b4ac65af460d0ad0 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:19.626855  1234 raft_consensus.cc:357] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:19.628626   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_055d241428174f7891615d2422293f39 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:19.629856  1234 raft_consensus.cc:383] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.630226  1234 raft_consensus.cc:738] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.631021  1234 consensus_queue.cc:260] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.636308  1229 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:19.642652  1234 ts_tablet_manager.cc:1428] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.000s
W20250628 07:52:19.644059  1062 leader_election.cc:343] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:19.644688  1062 leader_election.cc:343] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.661473  1234 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:19.645042  1062 leader_election.cc:304] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:19.672899  1527 raft_consensus.cc:2747] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:19.683506  1234 tablet_bootstrap.cc:654] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.682297  1230 raft_consensus.cc:357] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.683954  1230 raft_consensus.cc:383] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.684249  1230 raft_consensus.cc:738] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.691131   921 leader_election.cc:304] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:19.690824  1229 tablet_bootstrap.cc:654] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.699332  1527 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:19.699776  1527 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.702282  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:19.702906  1129 raft_consensus.cc:2466] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:19.704106  1536 raft_consensus.cc:2802] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:19.704543  1536 raft_consensus.cc:695] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:19.705330  1536 consensus_queue.cc:237] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.710543  1230 consensus_queue.cc:260] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:19.713328   921 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:19.713780   921 leader_election.cc:304] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:19.714857   921 leader_election.cc:304] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:19.715890  1569 raft_consensus.cc:2747] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:19.716442  1569 raft_consensus.cc:2802] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:19.716755  1569 raft_consensus.cc:491] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:19.717073  1569 raft_consensus.cc:3058] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.722270  1234 tablet_bootstrap.cc:492] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:19.722769  1234 ts_tablet_manager.cc:1397] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.061s	user 0.014s	sys 0.000s
I20250628 07:52:19.725205  1230 ts_tablet_manager.cc:1428] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.307s	user 0.007s	sys 0.001s
I20250628 07:52:19.725598  1234 raft_consensus.cc:357] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.726264  1230 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:19.726421  1234 raft_consensus.cc:383] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.701637  1527 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:19.741930  1229 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:19.742808  1229 ts_tablet_manager.cc:1397] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.109s	user 0.014s	sys 0.003s
I20250628 07:52:19.744717   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:19.745543   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:19.726740  1234 raft_consensus.cc:738] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.747529  1234 consensus_queue.cc:260] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:19.746639  1062 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.755038  1229 raft_consensus.cc:357] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.760156  1229 raft_consensus.cc:383] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:19.761341  1062 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:19.762020  1062 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:19.762233  1229 raft_consensus.cc:738] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.764941  1527 raft_consensus.cc:2747] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:19.771564  1234 ts_tablet_manager.cc:1428] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.048s	user 0.006s	sys 0.003s
I20250628 07:52:19.767032  1229 consensus_queue.cc:260] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.784518  1562 ts_tablet_manager.cc:1918] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:19.785044  1562 log.cc:1199] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:19.785027  1234 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:19.786144  1562 ts_tablet_manager.cc:1939] T c4a5d0deb0a049a2b6cc2ac5a8dd3c17 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:19.785727  1229 ts_tablet_manager.cc:1428] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.042s	user 0.009s	sys 0.000s
I20250628 07:52:19.798056  1229 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:19.812536   921 consensus_peers.cc:489] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:19.813413   657 debug-util.cc:398] Leaking SignalData structure 0x7b0800080020 after lost signal to thread 589
W20250628 07:52:19.814145   657 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac6c0 after lost signal to thread 660
W20250628 07:52:19.838052   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_07319c0694ca4307b243e8938d21939b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:19.758828  1569 raft_consensus.cc:513] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.594950   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]) successfully deleted
W20250628 07:52:19.841585   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 246630us
I20250628 07:52:19.843252   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:19.843613  1234 tablet_bootstrap.cc:654] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.843822   824 raft_consensus.cc:3058] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.857224   824 raft_consensus.cc:2466] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:19.858582   920 leader_election.cc:304] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:19.859820  1504 raft_consensus.cc:2802] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:19.862102  1504 raft_consensus.cc:695] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:19.862972  1504 consensus_queue.cc:237] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.865386  1229 tablet_bootstrap.cc:654] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.870586  1234 tablet_bootstrap.cc:492] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:19.870998  1234 ts_tablet_manager.cc:1397] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.086s	user 0.000s	sys 0.011s
I20250628 07:52:19.873201  1234 raft_consensus.cc:357] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.873835  1234 raft_consensus.cc:383] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.874092  1234 raft_consensus.cc:738] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.874663  1234 consensus_queue.cc:260] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.876631  1234 ts_tablet_manager.cc:1428] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.002s
I20250628 07:52:19.880954  1229 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:19.881357  1229 ts_tablet_manager.cc:1397] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.087s	user 0.012s	sys 0.000s
I20250628 07:52:19.883508  1229 raft_consensus.cc:357] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.892454  1536 consensus_queue.cc:1035] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:19.893843  1234 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:19.884092  1229 raft_consensus.cc:383] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:19.894659  1229 raft_consensus.cc:738] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:19.895226  1229 consensus_queue.cc:260] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:19.897255  1229 ts_tablet_manager.cc:1428] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.000s
I20250628 07:52:19.899055  1229 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:19.907850  1536 consensus_queue.cc:1035] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:19.911662  1229 tablet_bootstrap.cc:654] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.898139  1569 leader_election.cc:290] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:19.945415   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 16a8bee148664e948529d32c0cf2c793 (table default.Table_0adee4e1249840be952b0a99dba4c161 [id=f59d42b137cf4536a002585b728017f6], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 16a8bee148664e948529d32c0cf2c793 consensus state has no leader
I20250628 07:52:19.833376  1230 tablet_bootstrap.cc:654] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:19.854003  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1c65fa809b17418c85d4d72ae1e0ee3a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:19.956210  1127 raft_consensus.cc:3058] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:19.962502  1244 ts_tablet_manager.cc:1918] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:19.964134  1244 log.cc:1199] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/63348a64bb7044dbad9065afff179fbc
I20250628 07:52:19.964900  1244 ts_tablet_manager.cc:1939] T 63348a64bb7044dbad9065afff179fbc P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:19.967201  1127 raft_consensus.cc:2466] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:19.967478  1571 consensus_queue.cc:1035] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:19.971771  1062 consensus_peers.cc:489] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:19.972729  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 98543.5us
I20250628 07:52:19.989646  1562 tablet_replica.cc:331] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:19.997467  1234 tablet_bootstrap.cc:654] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.000247   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]) successfully deleted
I20250628 07:52:20.000515  1229 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:20.005000  1562 raft_consensus.cc:2241] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:20.022320  1229 ts_tablet_manager.cc:1397] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.123s	user 0.004s	sys 0.010s
I20250628 07:52:20.032009  1590 tablet.cc:1722] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:20.035554  1234 tablet_bootstrap.cc:492] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:20.036185  1234 ts_tablet_manager.cc:1397] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.143s	user 0.010s	sys 0.002s
I20250628 07:52:20.038805  1234 raft_consensus.cc:357] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.042407  1230 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:20.042753  1234 raft_consensus.cc:383] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.042881  1230 ts_tablet_manager.cc:1397] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.317s	user 0.006s	sys 0.008s
I20250628 07:52:20.043144  1234 raft_consensus.cc:738] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.043967  1234 consensus_queue.cc:260] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.006762  1562 raft_consensus.cc:2270] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:20.050952  1562 ts_tablet_manager.cc:1905] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:19.984823   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]) successfully deleted
W20250628 07:52:20.057474   533 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 72714.9us
I20250628 07:52:20.066999  1562 ts_tablet_manager.cc:1918] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20250628 07:52:20.067442  1562 log.cc:1199] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/63348a64bb7044dbad9065afff179fbc
I20250628 07:52:20.067852  1234 ts_tablet_manager.cc:1428] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.031s	user 0.005s	sys 0.001s
I20250628 07:52:20.068230  1562 ts_tablet_manager.cc:1939] T 63348a64bb7044dbad9065afff179fbc P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:20.071928  1504 raft_consensus.cc:491] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.072368  1504 raft_consensus.cc:513] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.074249  1504 leader_election.cc:290] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.076148   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.076714   825 raft_consensus.cc:2466] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.078294  1234 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:20.083209   920 leader_election.cc:304] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.087635  1504 raft_consensus.cc:2802] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:20.088011  1504 raft_consensus.cc:491] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:20.089159  1504 raft_consensus.cc:3058] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.024776  1229 raft_consensus.cc:357] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.097872  1229 raft_consensus.cc:383] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.098237  1229 raft_consensus.cc:738] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.099217  1229 consensus_queue.cc:260] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.115633  1234 tablet_bootstrap.cc:654] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.084478  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.121783  1126 raft_consensus.cc:2466] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.135660   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 63348a64bb7044dbad9065afff179fbc (table default.Table_78d794b2cff54bdf88d1996d140e4de8 [id=ff9dc70424f54129963f83df1d05463f]) successfully deleted
I20250628 07:52:20.119834  1230 raft_consensus.cc:357] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.136603  1230 raft_consensus.cc:383] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.138113  1230 raft_consensus.cc:738] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.138846  1230 consensus_queue.cc:260] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.101338  1229 ts_tablet_manager.cc:1428] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.079s	user 0.006s	sys 0.000s
I20250628 07:52:20.147006  1229 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:20.150750  1234 tablet_bootstrap.cc:492] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:20.151268  1234 ts_tablet_manager.cc:1397] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.074s	user 0.007s	sys 0.003s
I20250628 07:52:20.152546  1230 ts_tablet_manager.cc:1428] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.109s	user 0.004s	sys 0.001s
I20250628 07:52:20.153601  1234 raft_consensus.cc:357] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.154275  1234 raft_consensus.cc:383] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.154534  1234 raft_consensus.cc:738] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.159309  1229 tablet_bootstrap.cc:654] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.163841   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:20.166040  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.166990   544 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 158) took 1501 ms. Trace:
I20250628 07:52:20.167138   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.167815  1234 consensus_queue.cc:260] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:20.168629   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:20.170055  1230 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:20.176556   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:20.167292   544 rpcz_store.cc:276] 0628 07:52:18.665899 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:18.666257 (+   358us) service_pool.cc:225] Handling call
0628 07:52:18.718584 (+ 52327us) spinlock_profiling.cc:243] Waited 51.5 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea941046 00007f5be86601a4 00007f5be8660132 00007f5be8660061 00007f5be865ffdc 00007f5be865f292 00007f5be6957dcc 00007f5be6957206
0628 07:52:18.719519 (+   935us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:18.720053 (+   534us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:18.720165 (+   112us) catalog_manager.cc:2799] Locking tablets
0628 07:52:20.127507 (+1407342us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:20.163425 (+ 35918us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:20.163593 (+   168us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:20.163669 (+    76us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:20.166936 (+  3267us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.129947 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.130367 (+   420us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.130391 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:20.130404 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:20.130720 (+   316us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.130759 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.130780 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:20.131027 (+   247us) tablet.cc:699] Row locks acquired
0628 07:52:20.131041 (+    14us) write_op.cc:260] PREPARE: finished
0628 07:52:20.131141 (+   100us) write_op.cc:270] Start()
0628 07:52:20.131221 (+    80us) write_op.cc:276] Timestamp: P: 1751097140130927 usec, L: 0
0628 07:52:20.131238 (+    17us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.131719 (+   481us) log.cc:844] Serialized 731 byte log entry
0628 07:52:20.145472 (+ 13753us) op_driver.cc:464] REPLICATION: finished
0628 07:52:20.161871 (+ 16399us) write_op.cc:301] APPLY: starting
0628 07:52:20.161928 (+    57us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:20.162028 (+   100us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:20.162114 (+    86us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:20.162361 (+   247us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:20.162500 (+   139us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:20.162543 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:20.162715 (+   172us) log.cc:844] Serialized 29 byte log entry
0628 07:52:20.162845 (+   130us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.162908 (+    63us) write_op.cc:454] Released schema lock
0628 07:52:20.162944 (+    36us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:20.163139 (+   195us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.163161 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":118528256,"child_traces":[["op",{"apply.queue_time_us":16233,"apply.run_cpu_time_us":1248,"apply.run_wall_time_us":1248,"num_ops":2,"prepare.queue_time_us":749,"prepare.run_cpu_time_us":2027,"prepare.run_wall_time_us":2027,"raft.queue_time_us":5825,"replication_time_us":14141}]]}
I20250628 07:52:20.179651   544 master_service.cc:948] ReplaceTablet: received request to replace tablet 6e912d18dcff4a938a867ea06ee7e0f0 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:20.183311  1234 ts_tablet_manager.cc:1428] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.002s
I20250628 07:52:20.195561  1230 tablet_bootstrap.cc:654] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.200438   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:20.166666   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.199090  1571 raft_consensus.cc:491] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.201328  1571 raft_consensus.cc:513] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.203554  1571 leader_election.cc:290] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.205407  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.206156  1126 raft_consensus.cc:2466] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.207501   921 leader_election.cc:304] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.209787  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.210361   541 catalog_manager.cc:6652] ReplaceTablet: tablet 3223005021ae4e38b4ac65af460d0ad0 deleted and replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e
I20250628 07:52:20.210435  1562 tablet_replica.cc:331] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:20.211966  1234 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:20.212728   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.213558  1244 tablet_replica.cc:331] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:20.213546   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.214228  1244 raft_consensus.cc:2241] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:20.214401   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.214651  1244 raft_consensus.cc:2270] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:20.215581   920 leader_election.cc:343] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.211330   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_92381327fd4e479298ecf33ccdd4eccb" } modify_external_catalogs: true
I20250628 07:52:20.216365  1244 ts_tablet_manager.cc:1905] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:20.216539   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_92381327fd4e479298ecf33ccdd4eccb" } modify_external_catalogs: true
I20250628 07:52:20.212488  1562 raft_consensus.cc:2241] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:20.218259  1562 raft_consensus.cc:2270] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:20.221252  1571 raft_consensus.cc:2802] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:20.221568  1571 raft_consensus.cc:491] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:20.221869  1571 raft_consensus.cc:3058] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.226841  1562 ts_tablet_manager.cc:1905] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:20.228863   539 catalog_manager.cc:5582] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:20.232352   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
I20250628 07:52:20.237797   539 catalog_manager.cc:5582] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:20.249244  1234 tablet_bootstrap.cc:654] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.264181  1571 raft_consensus.cc:513] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.269881  1234 tablet_bootstrap.cc:492] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:20.270442  1234 ts_tablet_manager.cc:1397] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.059s	user 0.010s	sys 0.002s
I20250628 07:52:20.274243  1234 raft_consensus.cc:357] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.275087  1234 raft_consensus.cc:383] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.275385  1234 raft_consensus.cc:738] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.284548  1230 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:20.268343  1229 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:20.275651  1504 raft_consensus.cc:513] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.282943  1536 raft_consensus.cc:491] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.288568  1229 ts_tablet_manager.cc:1397] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.142s	user 0.009s	sys 0.000s
I20250628 07:52:20.288595  1536 raft_consensus.cc:513] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.290634  1536 leader_election.cc:290] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.290926  1229 raft_consensus.cc:357] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.291498  1229 raft_consensus.cc:383] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.291747  1229 raft_consensus.cc:738] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.292284  1229 consensus_queue.cc:260] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.295476  1234 consensus_queue.cc:260] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:20.296418   921 leader_election.cc:343] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:20.297935   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c84f2570b2da45d0b3ce6428723263a5" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.298498   825 raft_consensus.cc:2466] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.300093   920 leader_election.cc:304] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:20.301407  1569 raft_consensus.cc:2802] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:20.301719  1569 raft_consensus.cc:491] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:20.302021  1569 raft_consensus.cc:3058] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.284950  1230 ts_tablet_manager.cc:1397] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.115s	user 0.004s	sys 0.004s
I20250628 07:52:20.311151  1234 ts_tablet_manager.cc:1428] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.040s	user 0.006s	sys 0.002s
I20250628 07:52:20.311547  1569 raft_consensus.cc:513] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:20.315199   921 leader_election.cc:343] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:20.315915   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c84f2570b2da45d0b3ce6428723263a5" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:20.316360   825 raft_consensus.cc:3058] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.317453  1244 ts_tablet_manager.cc:1918] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:20.317776  1244 log.cc:1199] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:20.318435  1244 ts_tablet_manager.cc:1939] T 3223005021ae4e38b4ac65af460d0ad0 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:20.320175  1229 ts_tablet_manager.cc:1428] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.031s	user 0.005s	sys 0.000s
I20250628 07:52:20.308642  1230 raft_consensus.cc:357] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.320688  1230 raft_consensus.cc:383] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.320942  1230 raft_consensus.cc:738] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.322769   825 raft_consensus.cc:2466] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:20.324254  1230 consensus_queue.cc:260] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.326319  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:20.326767  1126 raft_consensus.cc:3058] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.326793   920 leader_election.cc:304] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:20.329489  1536 raft_consensus.cc:2802] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:20.330319  1234 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:20.341881  1126 raft_consensus.cc:2466] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:20.345077   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7e8d9640c74c8da5ac00adaf86f8a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:20.188912   727 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:20.351763   727 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:20.260203   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:20.288319   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
W20250628 07:52:20.354475   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 66201.5us
I20250628 07:52:20.356189   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]) successfully deleted
I20250628 07:52:20.357962   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1f0b83a168f54608ae33309446b7b2aa"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:20.358407   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.359086   962 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.359210   964 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:20.359364   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1f0b83a168f54608ae33309446b7b2aa in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:20.345006  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:20.361012  1129 raft_consensus.cc:3058] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.361848  1562 ts_tablet_manager.cc:1918] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:20.369827   921 leader_election.cc:304] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.351244  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.353868  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.356209  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.348647  1230 ts_tablet_manager.cc:1428] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.042s	user 0.004s	sys 0.001s
I20250628 07:52:20.378943  1230 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:20.379211   964 tablet_service.cc:1468] Processing CreateTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed (DEFAULT_TABLE table=default.Table_b86a1653daf740d9a4927425c711c0b2 [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.382128  1600 raft_consensus.cc:2802] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:20.382717  1600 raft_consensus.cc:695] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:20.383628  1600 consensus_queue.cc:237] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.345193   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:20.357263   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.357558   805 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.357893   803 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.375698  1234 tablet_bootstrap.cc:654] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.385558   825 raft_consensus.cc:3058] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.368748  1129 raft_consensus.cc:2466] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:20.387915   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
W20250628 07:52:20.388757   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:20.390218   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:20.391441   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 6e912d18dcff4a938a867ea06ee7e0f0
I20250628 07:52:20.392760   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 with delete_type TABLET_DATA_DELETED (replaced by tablet 83ffa65248fc4db3aecc5f91b648b83f) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.392890   544 catalog_manager.cc:6652] ReplaceTablet: tablet 6e912d18dcff4a938a867ea06ee7e0f0 deleted and replaced by tablet 83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:20.393463   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 with delete_type TABLET_DATA_DELETED (replaced by tablet 83ffa65248fc4db3aecc5f91b648b83f) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.393674   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_2af9142e039c4b78a79c8f8252e66c86"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:20.394294  1244 tablet_replica.cc:331] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:20.395032  1244 raft_consensus.cc:2241] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:20.395085   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_2af9142e039c4b78a79c8f8252e66c86 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:20.395598  1244 raft_consensus.cc:2270] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:20.396224  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 6e912d18dcff4a938a867ea06ee7e0f0 with delete_type TABLET_DATA_DELETED (replaced by tablet 83ffa65248fc4db3aecc5f91b648b83f) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:20.397302   921 leader_election.cc:304] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.399585  1562 log.cc:1199] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:20.399502  1504 leader_election.cc:290] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.399922  1457 raft_consensus.cc:491] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.400163  1504 raft_consensus.cc:491] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.400347  1457 raft_consensus.cc:513] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.401605  1562 ts_tablet_manager.cc:1939] T 3223005021ae4e38b4ac65af460d0ad0 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:20.400542  1504 raft_consensus.cc:513] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.403291  1504 leader_election.cc:290] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.403707  1504 raft_consensus.cc:2802] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:20.403468   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:20.404073  1504 raft_consensus.cc:695] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:20.405174  1571 leader_election.cc:290] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.288015  1594 tablet_replica.cc:331] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:20.405707   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]) successfully deleted
I20250628 07:52:20.406760  1594 raft_consensus.cc:2241] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:20.407349  1457 leader_election.cc:290] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.408027  1229 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:20.408546  1244 ts_tablet_manager.cc:1905] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:20.411406   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 532ba24d0d644313b9ef4e3ec10b0fed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.421274  1594 raft_consensus.cc:2270] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:20.422626  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e28ab08f878d4d4b86a5646f687b08fc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.423347  1129 raft_consensus.cc:2391] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:20.412989  1229 tablet_bootstrap.cc:654] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.424191   758 leader_election.cc:304] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:20.425401  1457 raft_consensus.cc:3058] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.430445  1230 tablet_bootstrap.cc:654] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.431073  1569 leader_election.cc:290] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.431653   805 tablet_service.cc:1468] Processing CreateTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed (DEFAULT_TABLE table=default.Table_b86a1653daf740d9a4927425c711c0b2 [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.432719   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 532ba24d0d644313b9ef4e3ec10b0fed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.433435  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.434497  1129 raft_consensus.cc:2466] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.435506   921 leader_election.cc:304] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.436527  1602 raft_consensus.cc:2802] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:20.436830  1602 raft_consensus.cc:491] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:20.437147  1602 raft_consensus.cc:3058] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.439241  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed (DEFAULT_TABLE table=default.Table_b86a1653daf740d9a4927425c711c0b2 [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.440344  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 532ba24d0d644313b9ef4e3ec10b0fed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.440379  1457 raft_consensus.cc:2747] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:20.444273   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
W20250628 07:52:20.445749   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:20.447255   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:20.448112  1562 tablet_replica.cc:331] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:20.450950   920 leader_election.cc:343] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.451990  1562 raft_consensus.cc:2241] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:20.452842  1562 raft_consensus.cc:2270] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:20.452867   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.453505   823 raft_consensus.cc:2466] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:20.454087   532 catalog_manager.cc:5001] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): alter failed for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]),no further retry: Not found: Tablet not found: c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:20.457644   962 tablet_service.cc:1468] Processing CreateTablet for tablet 14fedcf110e64c33978004743c5096ed (DEFAULT_TABLE table=default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 [id=63b87421f31d454dbc6999067f297582]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.458612   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 14fedcf110e64c33978004743c5096ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.464797  1234 tablet_bootstrap.cc:492] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:20.464962   921 consensus_peers.cc:489] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7a3e80af50b542ce94ea3cb2b2cef3b8. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:20.465217  1234 ts_tablet_manager.cc:1397] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.135s	user 0.012s	sys 0.003s
I20250628 07:52:20.467442  1234 raft_consensus.cc:357] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.468101  1234 raft_consensus.cc:383] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.468344  1234 raft_consensus.cc:738] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.468844  1562 ts_tablet_manager.cc:1905] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:20.486692   805 tablet_service.cc:1468] Processing CreateTablet for tablet 14fedcf110e64c33978004743c5096ed (DEFAULT_TABLE table=default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 [id=63b87421f31d454dbc6999067f297582]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.488082   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 14fedcf110e64c33978004743c5096ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.490412  1230 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:20.490850  1230 ts_tablet_manager.cc:1397] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.112s	user 0.008s	sys 0.002s
I20250628 07:52:20.492848   543 catalog_manager.cc:5582] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:20.494227   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:20.496784  1527 raft_consensus.cc:491] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.495995  1234 consensus_queue.cc:260] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.497242  1527 raft_consensus.cc:513] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.498458  1230 raft_consensus.cc:357] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.498960  1527 leader_election.cc:290] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:20.499065  1230 raft_consensus.cc:383] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.499336  1230 raft_consensus.cc:738] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.500612   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:20.503382  1062 leader_election.cc:343] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.503149  1230 consensus_queue.cc:260] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:20.510497   758 consensus_peers.cc:489] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:20.520359  1234 ts_tablet_manager.cc:1428] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.055s	user 0.004s	sys 0.000s
I20250628 07:52:20.523007  1230 ts_tablet_manager.cc:1428] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.001s
I20250628 07:52:20.528509  1230 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:20.528532   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:20.535331  1234 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:20.540714   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:20.542270  1244 ts_tablet_manager.cc:1918] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:20.542954   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:20.543751   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:20.544965   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:20.545644   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]). Expected version 3 got 0
I20250628 07:52:20.550879  1230 tablet_bootstrap.cc:654] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.554291  1562 ts_tablet_manager.cc:1918] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:20.554639  1562 log.cc:1199] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/6e912d18dcff4a938a867ea06ee7e0f0
I20250628 07:52:20.555287  1562 ts_tablet_manager.cc:1939] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:20.563261  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 14fedcf110e64c33978004743c5096ed (DEFAULT_TABLE table=default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 [id=63b87421f31d454dbc6999067f297582]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.564570  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 14fedcf110e64c33978004743c5096ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.566566  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 0f29c8ffb7df410295278eee9c578b03 (DEFAULT_TABLE table=default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.567564  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f29c8ffb7df410295278eee9c578b03. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:20.579644   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 124901us
I20250628 07:52:20.586673   538 catalog_manager.cc:5582] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:20.588121   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:20.561102   805 tablet_service.cc:1468] Processing CreateTablet for tablet 0f29c8ffb7df410295278eee9c578b03 (DEFAULT_TABLE table=default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.590005   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f29c8ffb7df410295278eee9c578b03. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.590423   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:20.591248   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:20.592083   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:20.593619   546 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 167) took 1063 ms. Trace:
I20250628 07:52:20.593904   546 rpcz_store.cc:276] 0628 07:52:19.530581 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:19.530994 (+   413us) service_pool.cc:225] Handling call
0628 07:52:19.560103 (+ 29109us) spinlock_profiling.cc:243] Waited 28.3 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea9413b6 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc 00007f5be6957206
0628 07:52:19.560883 (+   780us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:19.566517 (+  5634us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:19.567224 (+   707us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:19.567512 (+   288us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:19.634808 (+ 67296us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:20.593552 (+958744us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:19.594431 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:19.595264 (+   833us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:19.595332 (+    68us) write_op.cc:435] Acquired schema lock
0628 07:52:19.595350 (+    18us) tablet.cc:660] Decoding operations
0628 07:52:19.596027 (+   677us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:19.596208 (+   181us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:19.596298 (+    90us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:19.605238 (+  8940us) tablet.cc:699] Row locks acquired
0628 07:52:19.605257 (+    19us) write_op.cc:260] PREPARE: finished
0628 07:52:19.605386 (+   129us) write_op.cc:270] Start()
0628 07:52:19.605492 (+   106us) write_op.cc:276] Timestamp: P: 1751097139605057 usec, L: 0
0628 07:52:19.605507 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:19.606055 (+   548us) log.cc:844] Serialized 405 byte log entry
0628 07:52:19.633184 (+ 27129us) op_driver.cc:464] REPLICATION: finished
0628 07:52:19.633612 (+   428us) write_op.cc:301] APPLY: starting
0628 07:52:19.633673 (+    61us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:19.633763 (+    90us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:19.633858 (+    95us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:19.634052 (+   194us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:19.634199 (+   147us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:19.634239 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:19.634407 (+   168us) log.cc:844] Serialized 23 byte log entry
0628 07:52:19.634549 (+   142us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:19.634614 (+    65us) write_op.cc:454] Released schema lock
0628 07:52:19.634642 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:19.635131 (+   489us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:19.635163 (+    32us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":65195904,"child_traces":[["op",{"apply.queue_time_us":277,"apply.run_cpu_time_us":1227,"apply.run_wall_time_us":1492,"num_ops":1,"prepare.queue_time_us":25736,"prepare.run_cpu_time_us":2596,"prepare.run_wall_time_us":33557,"raft.queue_time_us":3174,"raft.run_cpu_time_us":749,"raft.run_wall_time_us":1152,"replication_time_us":27544}]]}
I20250628 07:52:20.599258   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_915f3db7b34449c4b8124af06f772bc8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:20.601123   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_915f3db7b34449c4b8124af06f772bc8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:20.601253  1244 log.cc:1199] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/6e912d18dcff4a938a867ea06ee7e0f0
I20250628 07:52:20.602203  1244 ts_tablet_manager.cc:1939] T 6e912d18dcff4a938a867ea06ee7e0f0 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:20.602771   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:20.603479   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:20.604908  1234 tablet_bootstrap.cc:654] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.579588  1536 raft_consensus.cc:695] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:20.611531  1230 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:20.612676  1230 ts_tablet_manager.cc:1397] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.084s	user 0.008s	sys 0.003s
I20250628 07:52:20.610548   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:20.614441  1062 leader_election.cc:343] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.614837  1062 leader_election.cc:304] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:20.615799  1527 raft_consensus.cc:2747] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:20.616963  1457 consensus_queue.cc:1035] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:20.618747   961 tablet_service.cc:1468] Processing CreateTablet for tablet 0f29c8ffb7df410295278eee9c578b03 (DEFAULT_TABLE table=default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.404697  1504 consensus_queue.cc:237] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.621882  1230 raft_consensus.cc:357] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.627118  1230 raft_consensus.cc:383] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.627370  1230 raft_consensus.cc:738] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.626199  1594 ts_tablet_manager.cc:1905] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:20.628692  1230 consensus_queue.cc:260] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.632391  1601 consensus_queue.cc:1035] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.006s
I20250628 07:52:20.636373  1536 consensus_queue.cc:237] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.645394  1527 raft_consensus.cc:491] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.646003  1527 raft_consensus.cc:513] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.648339  1527 leader_election.cc:290] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:20.649576  1230 ts_tablet_manager.cc:1428] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.036s	user 0.005s	sys 0.000s
I20250628 07:52:20.650981   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:20.652477  1062 leader_election.cc:343] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.655602   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:20.655787  1601 consensus_queue.cc:1035] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:20.656937  1062 leader_election.cc:343] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.657299  1062 leader_election.cc:304] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:20.632886   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0f29c8ffb7df410295278eee9c578b03. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.658197  1527 raft_consensus.cc:2747] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:20.660643  1230 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:20.663396   825 raft_consensus.cc:2466] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:20.676435  1230 tablet_bootstrap.cc:654] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.679199   542 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 163) took 1699 ms. Trace:
I20250628 07:52:20.687516  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:20.689061   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c84f2570b2da45d0b3ce6428723263a5 because the tablet was not found. No further retry: Not found: Tablet not found: c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:20.690492  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:20.692687   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because the tablet was not found. No further retry: Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:20.679553   542 rpcz_store.cc:276] 0628 07:52:18.980061 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:18.990795 (+ 10734us) service_pool.cc:225] Handling call
0628 07:52:19.065975 (+ 75180us) spinlock_profiling.cc:243] Waited 74 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea9413b6 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc 00007f5be6957206
0628 07:52:19.066780 (+   805us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:19.067382 (+   602us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:19.068037 (+   655us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:19.068241 (+   204us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:19.133736 (+ 65495us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:20.679131 (+1545395us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:19.070492 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:19.070856 (+   364us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:19.070876 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:19.070887 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:19.071125 (+   238us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:19.071157 (+    32us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:19.071176 (+    19us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:19.071303 (+   127us) tablet.cc:699] Row locks acquired
0628 07:52:19.071314 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:19.071408 (+    94us) write_op.cc:270] Start()
0628 07:52:19.071481 (+    73us) write_op.cc:276] Timestamp: P: 1751097139071297 usec, L: 0
0628 07:52:19.071495 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:19.071924 (+   429us) log.cc:844] Serialized 405 byte log entry
0628 07:52:19.131608 (+ 59684us) op_driver.cc:464] REPLICATION: finished
0628 07:52:19.132395 (+   787us) write_op.cc:301] APPLY: starting
0628 07:52:19.132440 (+    45us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:19.132530 (+    90us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:19.132611 (+    81us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:19.132754 (+   143us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:19.132872 (+   118us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:19.132911 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:19.133318 (+   407us) log.cc:844] Serialized 23 byte log entry
0628 07:52:19.133472 (+   154us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:19.133520 (+    48us) write_op.cc:454] Released schema lock
0628 07:52:19.133548 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:19.134138 (+   590us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:19.134174 (+    36us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":170191232,"child_traces":[["op",{"apply.queue_time_us":653,"apply.run_cpu_time_us":1161,"apply.run_wall_time_us":1687,"num_ops":1,"prepare.queue_time_us":92,"prepare.run_cpu_time_us":1601,"prepare.run_wall_time_us":1600,"replication_time_us":60023}]]}
I20250628 07:52:20.696695   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:20.697715  1620 tablet.cc:1722] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:20.698765   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:20.706458   540 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 36) took 2834 ms (client timeout 1000 ms). Trace:
W20250628 07:52:20.711016   532 catalog_manager.cc:5001] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): alter failed for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 (table default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]),no further retry: Not found: Tablet not found: c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:20.718784  1527 raft_consensus.cc:491] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.719247  1527 raft_consensus.cc:513] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.720130   805 tablet_service.cc:1468] Processing CreateTablet for tablet ef5d1e5d358e424d818afeeebab3cfa2 (DEFAULT_TABLE table=default.Table_7dc8fa39558f4514bbe045fbd9c99910 [id=e5c2cd9d2c1b44499a609cc74538868b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.721094  1527 leader_election.cc:290] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:20.721338   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef5d1e5d358e424d818afeeebab3cfa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.722646   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "16a8bee148664e948529d32c0cf2c793" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:20.723886  1062 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.727521   962 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.727909  1108 tablet_service.cc:1468] Processing CreateTablet for tablet ef5d1e5d358e424d818afeeebab3cfa2 (DEFAULT_TABLE table=default.Table_7dc8fa39558f4514bbe045fbd9c99910 [id=e5c2cd9d2c1b44499a609cc74538868b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.728960  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef5d1e5d358e424d818afeeebab3cfa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.657519  1602 raft_consensus.cc:513] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.739317   803 tablet_service.cc:1468] Processing CreateTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb (DEFAULT_TABLE table=default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.740263   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f42b22f7096e46cf9cc09d1caecc9efb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.745601   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.749846  1615 raft_consensus.cc:491] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.750375  1615 raft_consensus.cc:513] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.751776   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.752681   962 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.752996   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "16a8bee148664e948529d32c0cf2c793" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:20.753226  1615 leader_election.cc:290] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.753533  1229 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:20.753511   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.753965  1229 ts_tablet_manager.cc:1397] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.346s	user 0.015s	sys 0.024s
I20250628 07:52:20.754873   962 tablet_service.cc:1468] Processing CreateTablet for tablet ef5d1e5d358e424d818afeeebab3cfa2 (DEFAULT_TABLE table=default.Table_7dc8fa39558f4514bbe045fbd9c99910 [id=e5c2cd9d2c1b44499a609cc74538868b]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:20.758275  1062 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.758709  1062 leader_election.cc:304] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:20.761425  1527 raft_consensus.cc:2747] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:20.765198   758 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:20.767338  1229 raft_consensus.cc:357] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.768103  1229 raft_consensus.cc:383] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.768374  1229 raft_consensus.cc:738] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.750702  1601 raft_consensus.cc:491] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.770179  1229 consensus_queue.cc:260] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.769424   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef5d1e5d358e424d818afeeebab3cfa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.770707  1601 raft_consensus.cc:513] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.775938  1229 ts_tablet_manager.cc:1428] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.022s	user 0.005s	sys 0.001s
I20250628 07:52:20.778008  1601 leader_election.cc:290] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.778980   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.779381  1602 leader_election.cc:290] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.779964   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:20.780222   920 leader_election.cc:343] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.780431   824 raft_consensus.cc:3058] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.785437   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d39d14572c4647c7b4880c71a8707c08" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:20.786008  1229 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:20.787884  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "de7ff3260b0049d682dba82f914474de" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:20.788517  1127 raft_consensus.cc:3058] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.789175   824 raft_consensus.cc:2466] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:20.787914  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.793722  1128 raft_consensus.cc:2466] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.798641  1127 raft_consensus.cc:2466] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:20.706921   540 rpcz_store.cc:269] 0628 07:52:17.880992 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:17.895264 (+ 14272us) service_pool.cc:225] Handling call
0628 07:52:20.706271 (+2811007us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:18.915971 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:18.916356 (+   385us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:18.916388 (+    32us) write_op.cc:435] Acquired schema lock
0628 07:52:18.916398 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:18.916715 (+   317us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:18.916742 (+    27us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:18.916760 (+    18us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:18.916875 (+   115us) tablet.cc:699] Row locks acquired
0628 07:52:18.916885 (+    10us) write_op.cc:260] PREPARE: finished
0628 07:52:18.917025 (+   140us) write_op.cc:270] Start()
0628 07:52:18.923214 (+  6189us) write_op.cc:276] Timestamp: P: 1751097138916932 usec, L: 0
0628 07:52:18.923233 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:18.923725 (+   492us) log.cc:844] Serialized 533 byte log entry
0628 07:52:19.072684 (+148959us) op_driver.cc:464] REPLICATION: finished
0628 07:52:19.074580 (+  1896us) write_op.cc:301] APPLY: starting
0628 07:52:19.074623 (+    43us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:19.074723 (+   100us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:19.074823 (+   100us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:19.075021 (+   198us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:19.075146 (+   125us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:19.075181 (+    35us) write_op.cc:312] APPLY: finished
0628 07:52:19.075317 (+   136us) log.cc:844] Serialized 23 byte log entry
0628 07:52:19.075466 (+   149us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:19.075514 (+    48us) write_op.cc:454] Released schema lock
0628 07:52:19.075541 (+    27us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":16492,"reported_tablets":14,"child_traces":[["op",{"apply.queue_time_us":1756,"apply.run_cpu_time_us":1200,"apply.run_wall_time_us":14671,"num_ops":1,"prepare.queue_time_us":381951,"prepare.run_cpu_time_us":1935,"prepare.run_wall_time_us":8177,"raft.queue_time_us":80,"raft.run_cpu_time_us":10,"raft.run_wall_time_us":11,"replication_time_us":149380}]]}
I20250628 07:52:20.815476   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0adee4e1249840be952b0a99dba4c161" } modify_external_catalogs: true
I20250628 07:52:20.816156   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0adee4e1249840be952b0a99dba4c161" } modify_external_catalogs: true
I20250628 07:52:20.821297   920 leader_election.cc:304] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:20.786027   986 raft_consensus.cc:2466] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:20.834002  1618 tablet.cc:1722] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:20.855007   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 34) took 1974 ms (client timeout 1000 ms). Trace:
I20250628 07:52:20.859395   964 tablet_service.cc:1468] Processing CreateTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb (DEFAULT_TABLE table=default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.868708   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f42b22f7096e46cf9cc09d1caecc9efb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:20.868855  1108 tablet_service.cc:1468] Processing CreateTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb (DEFAULT_TABLE table=default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:20.869055  1569 raft_consensus.cc:2802] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:20.869563  1569 raft_consensus.cc:695] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:20.870034  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f42b22f7096e46cf9cc09d1caecc9efb. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:20.869702   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 67081.5us
I20250628 07:52:20.870342  1569 consensus_queue.cc:237] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:20.853983   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 51063.9us
I20250628 07:52:20.899252   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:20.899226  1600 raft_consensus.cc:491] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.904234  1600 raft_consensus.cc:513] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.906209  1600 leader_election.cc:290] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:20.906570   921 leader_election.cc:304] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:20.913744  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:20.914306  1127 raft_consensus.cc:2466] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:20.879035   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 2f7e2b290c69467aa42ca6328307bc3c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:20.890290   757 leader_election.cc:304] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:20.916558  1234 tablet_bootstrap.cc:492] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:20.919680  1234 ts_tablet_manager.cc:1397] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.385s	user 0.018s	sys 0.018s
I20250628 07:52:20.924053  1626 raft_consensus.cc:2802] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:20.947507  1626 raft_consensus.cc:491] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:20.947608  1623 raft_consensus.cc:491] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:20.947878  1626 raft_consensus.cc:3058] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:20.948051  1623 raft_consensus.cc:513] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.950239  1623 leader_election.cc:290] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:20.952098   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:20.953308  1062 leader_election.cc:343] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:20.955237   822 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:20.955854   822 raft_consensus.cc:2466] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:20.965545   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2f7e2b290c69467aa42ca6328307bc3c because the tablet was not found. No further retry: Not found: Tablet not found: 2f7e2b290c69467aa42ca6328307bc3c
W20250628 07:52:20.970768   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 16a8bee148664e948529d32c0cf2c793 (table default.Table_0adee4e1249840be952b0a99dba4c161 [id=f59d42b137cf4536a002585b728017f6], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 16a8bee148664e948529d32c0cf2c793 consensus state has no leader
I20250628 07:52:20.972630  1234 raft_consensus.cc:357] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.973343  1234 raft_consensus.cc:383] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:20.973666  1234 raft_consensus.cc:738] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:20.974429  1234 consensus_queue.cc:260] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.976702  1234 ts_tablet_manager.cc:1428] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.057s	user 0.004s	sys 0.000s
I20250628 07:52:20.982657   541 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 173) took 1139 ms. Trace:
I20250628 07:52:20.992339  1234 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:20.990929  1626 raft_consensus.cc:513] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:20.983063   541 rpcz_store.cc:276] 0628 07:52:19.843466 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.357577 (+514111us) service_pool.cc:225] Handling call
0628 07:52:20.359711 (+  2134us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:20.460165 (+100454us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:20.982053 (+521888us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:20.982339 (+   286us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:20.982596 (+   257us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.576387 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.576772 (+   385us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.576795 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:20.576807 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:20.577060 (+   253us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.577133 (+    73us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.577155 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:20.577338 (+   183us) tablet.cc:699] Row locks acquired
0628 07:52:20.577350 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:20.577452 (+   102us) write_op.cc:270] Start()
0628 07:52:20.577528 (+    76us) write_op.cc:276] Timestamp: P: 1751097140577271 usec, L: 0
0628 07:52:20.577544 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.577990 (+   446us) log.cc:844] Serialized 433 byte log entry
0628 07:52:20.974641 (+396651us) op_driver.cc:464] REPLICATION: finished
0628 07:52:20.975269 (+   628us) write_op.cc:301] APPLY: starting
0628 07:52:20.975327 (+    58us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:20.975467 (+   140us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:20.975592 (+   125us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:20.976109 (+   517us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:20.976330 (+   221us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:20.976395 (+    65us) write_op.cc:312] APPLY: finished
0628 07:52:20.976587 (+   192us) log.cc:844] Serialized 29 byte log entry
0628 07:52:20.976725 (+   138us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.976780 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:20.976814 (+    34us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:20.977057 (+   243us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.977110 (+    53us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":487,"apply.run_cpu_time_us":1740,"apply.run_wall_time_us":1739,"num_ops":2,"prepare.queue_time_us":292,"prepare.run_cpu_time_us":1859,"prepare.run_wall_time_us":1859,"raft.queue_time_us":1010,"raft.run_cpu_time_us":12,"raft.run_wall_time_us":16,"replication_time_us":396997}]]}
W20250628 07:52:20.855500   543 rpcz_store.cc:269] 0628 07:52:18.880383 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:18.986765 (+106382us) service_pool.cc:225] Handling call
0628 07:52:19.062574 (+ 75809us) spinlock_profiling.cc:243] Waited 72.1 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:20.524247 (+1461673us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.524392 (+   145us) write_op.cc:454] Released schema lock
0628 07:52:20.854923 (+330531us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.496559 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.496935 (+   376us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.496958 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:20.496970 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:20.497240 (+   270us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.497279 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.497299 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:20.497428 (+   129us) tablet.cc:699] Row locks acquired
0628 07:52:20.497440 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:20.497537 (+    97us) write_op.cc:270] Start()
0628 07:52:20.497614 (+    77us) write_op.cc:276] Timestamp: P: 1751097140497363 usec, L: 0
0628 07:52:20.497628 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.498078 (+   450us) log.cc:844] Serialized 533 byte log entry
0628 07:52:20.512177 (+ 14099us) op_driver.cc:464] REPLICATION: finished
0628 07:52:20.513816 (+  1639us) write_op.cc:301] APPLY: starting
0628 07:52:20.513870 (+    54us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:20.515149 (+  1279us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:20.515237 (+    88us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:20.515416 (+   179us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:20.515537 (+   121us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:20.515578 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:20.515761 (+   183us) log.cc:844] Serialized 23 byte log entry
0628 07:52:20.515884 (+   123us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.515941 (+    57us) write_op.cc:454] Released schema lock
0628 07:52:20.515971 (+    30us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"reported_tablets":10,"spinlock_wait_cycles":165761408,"child_traces":[["op",{"apply.queue_time_us":1493,"apply.run_cpu_time_us":1143,"apply.run_wall_time_us":2319,"num_ops":1,"prepare.queue_time_us":119,"prepare.run_cpu_time_us":1790,"prepare.run_wall_time_us":1799,"raft.queue_time_us":1085,"raft.run_cpu_time_us":638,"raft.run_wall_time_us":638,"replication_time_us":14452}]]}
W20250628 07:52:21.012558   758 leader_election.cc:343] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:21.013953  1626 leader_election.cc:290] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.023911   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 0ac2a2d045864749841a9b5de03d140c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:11 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.024930  1234 tablet_bootstrap.cc:654] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:21.029138  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 (DEFAULT_TABLE table=default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.030143  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a248bf8a62b4b4eb206c0daff5e0568. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.043429  1618 tablet.cc:1722] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:21.025451   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0ac2a2d045864749841a9b5de03d140c because the tablet was not found. No further retry: Not found: Tablet not found: 0ac2a2d045864749841a9b5de03d140c
I20250628 07:52:21.047274   803 tablet_service.cc:1468] Processing CreateTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 (DEFAULT_TABLE table=default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.048228   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a248bf8a62b4b4eb206c0daff5e0568. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.054132   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.056459   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 474cec4459954e10a18de031946a2346 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.058120   805 tablet_service.cc:1515] Processing DeleteTablet for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.059690   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 84a8065c2d1349b792308ac3aec1a8fe with delete_type TABLET_DATA_DELETED (replaced by tablet ef060774191146168ee8d1e13c9cd71b) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.061269   805 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.064548   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 2df135033ee546f8bb9f4c4b54755b06 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.070039   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f22700fc70d4f14ae51a834900db22b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.071388  1229 tablet_bootstrap.cc:700] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Time spent opening tablet: real 0.134s	user 0.001s	sys 0.001s
I20250628 07:52:21.071729   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:21.072105  1229 tablet_bootstrap.cc:654] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:20.931003  1536 raft_consensus.cc:2802] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:21.074750   805 tablet_service.cc:1515] Processing DeleteTablet for tablet deb5a9ae7ab5463cae35ff1086fd8348 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.076766  1230 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:21.077224  1230 ts_tablet_manager.cc:1397] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.417s	user 0.008s	sys 0.032s
I20250628 07:52:21.077195  1234 tablet_bootstrap.cc:492] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:21.077796  1234 ts_tablet_manager.cc:1397] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.086s	user 0.012s	sys 0.001s
I20250628 07:52:21.080181   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 67a81eb4342740e5bc16ee7f5eacb489 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:12 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.080507  1230 raft_consensus.cc:357] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.084285  1230 raft_consensus.cc:383] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.084537  1230 raft_consensus.cc:738] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.086194  1230 consensus_queue.cc:260] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.089605   805 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.090291  1244 tablet_replica.cc:331] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:21.091343  1244 raft_consensus.cc:2241] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:21.092152  1244 raft_consensus.cc:2270] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:21.094492  1244 ts_tablet_manager.cc:1905] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:21.097044  1234 raft_consensus.cc:357] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.097729  1234 raft_consensus.cc:383] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.098006  1234 raft_consensus.cc:738] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:21.102283   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 3223005021ae4e38b4ac65af460d0ad0 because the tablet was not found. No further retry: Not found: Tablet not found: 3223005021ae4e38b4ac65af460d0ad0
W20250628 07:52:21.103066   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 474cec4459954e10a18de031946a2346 because the tablet was not found. No further retry: Not found: Tablet not found: 474cec4459954e10a18de031946a2346
I20250628 07:52:21.102846  1234 consensus_queue.cc:260] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:21.103809   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet e71a4b8cfa3a41d1837ca6fd2e2a3e67 because the tablet was not found. No further retry: Not found: Tablet not found: e71a4b8cfa3a41d1837ca6fd2e2a3e67
W20250628 07:52:21.104511   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 84a8065c2d1349b792308ac3aec1a8fe because the tablet was not found. No further retry: Not found: Tablet not found: 84a8065c2d1349b792308ac3aec1a8fe
W20250628 07:52:21.105437   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 because the tablet was not found. No further retry: Not found: Tablet not found: c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:21.108026  1234 ts_tablet_manager.cc:1428] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.030s	user 0.002s	sys 0.004s
I20250628 07:52:21.111521  1234 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:21.136462  1234 tablet_bootstrap.cc:654] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:21.141445  1062 leader_election.cc:343] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.141888  1062 leader_election.cc:304] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:21.143568   964 tablet_service.cc:1468] Processing CreateTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 (DEFAULT_TABLE table=default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.144742   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a248bf8a62b4b4eb206c0daff5e0568. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.154826  1234 tablet_bootstrap.cc:492] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:21.155228  1234 ts_tablet_manager.cc:1397] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.044s	user 0.007s	sys 0.003s
I20250628 07:52:21.157769  1234 raft_consensus.cc:357] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.158370  1234 raft_consensus.cc:383] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.158699  1234 raft_consensus.cc:738] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.159322  1234 consensus_queue.cc:260] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.108210  1536 raft_consensus.cc:491] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:21.166623  1234 ts_tablet_manager.cc:1428] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.011s	user 0.005s	sys 0.003s
I20250628 07:52:21.172708  1234 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:21.186698  1527 raft_consensus.cc:2747] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:21.187865  1620 tablet.cc:1722] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:21.190092  1230 ts_tablet_manager.cc:1428] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.113s	user 0.005s	sys 0.000s
I20250628 07:52:21.189924  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.190913  1230 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:21.190933  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba2a10acdf0462bb05628351cdbbbe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.192019  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.194980  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 with delete_type TABLET_DATA_DELETED (replaced by tablet 8feee3be1a664cae9da417b190163a94) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.196296  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.184343  1234 tablet_bootstrap.cc:654] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:21.203852  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.204773  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fa4299a9e0f4b409d04c8c5900901c0. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:21.108347   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 63770.9us
I20250628 07:52:21.139106  1601 consensus_queue.cc:1035] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.047s
I20250628 07:52:21.168354  1536 raft_consensus.cc:3058] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:21.108476   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 63660.7us
I20250628 07:52:21.139240  1600 consensus_queue.cc:1035] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.047s
I20250628 07:52:21.224212  1230 tablet_bootstrap.cc:654] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:21.235105   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 129687us
W20250628 07:52:21.115536   531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111) [suppressed 7 similar messages]
I20250628 07:52:21.121300   545 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3] alter complete (version 3)
W20250628 07:52:21.237723   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 36) took 1735 ms (client timeout 1000 ms). Trace:
W20250628 07:52:21.238046   545 rpcz_store.cc:269] 0628 07:52:19.502177 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:19.502320 (+   143us) service_pool.cc:225] Handling call
0628 07:52:19.571884 (+ 69564us) spinlock_profiling.cc:243] Waited 67.3 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.237662 (+1665778us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.702221 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.702602 (+   381us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.702629 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:20.702643 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:20.702887 (+   244us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.702923 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.702957 (+    34us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:20.703096 (+   139us) tablet.cc:699] Row locks acquired
0628 07:52:20.703111 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:20.703213 (+   102us) write_op.cc:270] Start()
0628 07:52:20.703292 (+    79us) write_op.cc:276] Timestamp: P: 1751097140703018 usec, L: 0
0628 07:52:20.703310 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.703774 (+   464us) log.cc:844] Serialized 356 byte log entry
0628 07:52:21.116493 (+412719us) op_driver.cc:464] REPLICATION: finished
0628 07:52:21.118942 (+  2449us) write_op.cc:301] APPLY: starting
0628 07:52:21.119006 (+    64us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:21.119117 (+   111us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:21.119231 (+   114us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:21.119423 (+   192us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:21.119570 (+   147us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:21.119612 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:21.119753 (+   141us) log.cc:844] Serialized 23 byte log entry
0628 07:52:21.119892 (+   139us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.119973 (+    81us) write_op.cc:454] Released schema lock
0628 07:52:21.120012 (+    39us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"reported_tablets":8,"spinlock_wait_cycles":154867072,"child_traces":[["op",{"apply.queue_time_us":2329,"apply.run_cpu_time_us":1310,"apply.run_wall_time_us":2829,"num_ops":1,"prepare.queue_time_us":116,"prepare.run_cpu_time_us":1843,"prepare.run_wall_time_us":2999,"raft.queue_time_us":95,"raft.run_cpu_time_us":18,"raft.run_wall_time_us":18,"replication_time_us":413106}]]}
I20250628 07:52:21.122854   544 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 174) took 1075 ms. Trace:
I20250628 07:52:21.241184   544 rpcz_store.cc:276] 0628 07:52:20.047890 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.393319 (+345429us) service_pool.cc:225] Handling call
0628 07:52:20.395442 (+  2123us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:20.572168 (+176726us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:21.121971 (+549803us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:21.122456 (+   485us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:21.122781 (+   325us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.658099 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.658521 (+   422us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.658543 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:20.658555 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:20.658832 (+   277us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.658882 (+    50us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.658916 (+    34us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:20.659119 (+   203us) tablet.cc:699] Row locks acquired
0628 07:52:20.659129 (+    10us) write_op.cc:260] PREPARE: finished
0628 07:52:20.659226 (+    97us) write_op.cc:270] Start()
0628 07:52:20.659300 (+    74us) write_op.cc:276] Timestamp: P: 1751097140659037 usec, L: 0
0628 07:52:20.659315 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.659801 (+   486us) log.cc:844] Serialized 433 byte log entry
0628 07:52:21.116257 (+456456us) op_driver.cc:464] REPLICATION: finished
0628 07:52:21.116681 (+   424us) write_op.cc:301] APPLY: starting
0628 07:52:21.116745 (+    64us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:21.116888 (+   143us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:21.117032 (+   144us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:21.117567 (+   535us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:21.117753 (+   186us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:21.117816 (+    63us) write_op.cc:312] APPLY: finished
0628 07:52:21.118058 (+   242us) log.cc:844] Serialized 29 byte log entry
0628 07:52:21.118261 (+   203us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.118344 (+    83us) write_op.cc:454] Released schema lock
0628 07:52:21.118387 (+    43us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:21.118650 (+   263us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.118685 (+    35us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":285,"apply.run_cpu_time_us":1911,"apply.run_wall_time_us":1921,"num_ops":2,"prepare.queue_time_us":86,"prepare.run_cpu_time_us":2001,"prepare.run_wall_time_us":3262,"raft.queue_time_us":1161,"raft.run_cpu_time_us":12,"raft.run_wall_time_us":13,"replication_time_us":456848}]]}
I20250628 07:52:21.244424   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6b7424c1f64e4d9a90a106ee7637dac0" } new_table_name: "default.Table_9fc4e9c7ab7d413cba8e997e53c22844" modify_external_catalogs: true
I20250628 07:52:21.154353   542 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 176) took 1018 ms. Trace:
I20250628 07:52:21.248710   542 rpcz_store.cc:276] 0628 07:52:20.136136 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.696171 (+560035us) service_pool.cc:225] Handling call
0628 07:52:20.699192 (+  3021us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:20.741218 (+ 42026us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:21.153522 (+412304us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:21.153883 (+   361us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:21.154272 (+   389us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.860466 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.860900 (+   434us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.860930 (+    30us) write_op.cc:435] Acquired schema lock
0628 07:52:20.860960 (+    30us) tablet.cc:660] Decoding operations
0628 07:52:20.861407 (+   447us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.861452 (+    45us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.861482 (+    30us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:20.861699 (+   217us) tablet.cc:699] Row locks acquired
0628 07:52:20.861716 (+    17us) write_op.cc:260] PREPARE: finished
0628 07:52:20.861830 (+   114us) write_op.cc:270] Start()
0628 07:52:20.861924 (+    94us) write_op.cc:276] Timestamp: P: 1751097140861616 usec, L: 0
0628 07:52:20.861944 (+    20us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.862563 (+   619us) log.cc:844] Serialized 433 byte log entry
0628 07:52:21.142791 (+280228us) op_driver.cc:464] REPLICATION: finished
0628 07:52:21.145658 (+  2867us) write_op.cc:301] APPLY: starting
0628 07:52:21.145735 (+    77us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:21.145865 (+   130us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:21.145995 (+   130us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:21.146464 (+   469us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:21.146638 (+   174us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:21.146698 (+    60us) write_op.cc:312] APPLY: finished
0628 07:52:21.146900 (+   202us) log.cc:844] Serialized 29 byte log entry
0628 07:52:21.153235 (+  6335us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.153296 (+    61us) write_op.cc:454] Released schema lock
0628 07:52:21.153328 (+    32us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2731,"apply.run_cpu_time_us":1759,"apply.run_wall_time_us":9223,"num_ops":2,"prepare.queue_time_us":117992,"prepare.run_cpu_time_us":2240,"prepare.run_wall_time_us":3398,"replication_time_us":280732}]]}
I20250628 07:52:21.252545   542 master_service.cc:948] ReplaceTablet: received request to replace tablet 1642348a16574345b31cebda23575102 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:21.255553  1230 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:21.240978   545 master_service.cc:948] ReplaceTablet: received request to replace tablet 83f8dcff296a4429a5f16bde59c8ebee from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:21.255940  1230 ts_tablet_manager.cc:1397] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.065s	user 0.005s	sys 0.003s
I20250628 07:52:21.258107  1230 raft_consensus.cc:357] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.258705  1230 raft_consensus.cc:383] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.258953  1230 raft_consensus.cc:738] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.183907   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_bcf5c3bfda174712bc8205e0bbbfe24b" } modify_external_catalogs: true
I20250628 07:52:21.262333   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_bcf5c3bfda174712bc8205e0bbbfe24b" } modify_external_catalogs: true
W20250628 07:52:21.272231   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 37) took 1463 ms (client timeout 1000 ms). Trace:
W20250628 07:52:21.272609   547 rpcz_store.cc:269] 0628 07:52:19.808739 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:19.828963 (+ 20224us) service_pool.cc:225] Handling call
0628 07:52:19.875474 (+ 46511us) spinlock_profiling.cc:243] Waited 43.8 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.272168 (+1396694us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":17,"spinlock_wait_cycles":100686720}
I20250628 07:52:21.273936   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4a45d5758c0f40209ba04b7d96752a10" } new_table_name: "default.Table_1935e83524ad453c8175710332a3c459" modify_external_catalogs: true
I20250628 07:52:21.282238  1640 tablet_replica.cc:331] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:21.283432  1640 raft_consensus.cc:2241] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:21.284065  1640 raft_consensus.cc:2270] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:21.286275   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.AlterSchema -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 50676.6us
W20250628 07:52:21.287050   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2df135033ee546f8bb9f4c4b54755b06 because the tablet was not found. No further retry: Not found: Tablet not found: 2df135033ee546f8bb9f4c4b54755b06
I20250628 07:52:21.287576  1234 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:21.218611   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 35) took 2738 ms (client timeout 1000 ms). Trace:
W20250628 07:52:21.220402   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 3223005021ae4e38b4ac65af460d0ad0 because the tablet was not found. No further retry: Not found: Tablet not found: 3223005021ae4e38b4ac65af460d0ad0
W20250628 07:52:21.289374   533 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 68999us
I20250628 07:52:21.289836   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d39d14572c4647c7b4880c71a8707c08" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:21.290517   986 raft_consensus.cc:3058] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:21.290722   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 because the tablet was not found. No further retry: Not found: Tablet not found: c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:21.231840   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 16a8bee148664e948529d32c0cf2c793
I20250628 07:52:21.297340  1234 ts_tablet_manager.cc:1397] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.125s	user 0.014s	sys 0.009s
I20250628 07:52:21.299624  1234 raft_consensus.cc:357] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.300060  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 16a8bee148664e948529d32c0cf2c793 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.300288  1234 raft_consensus.cc:383] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.300514  1234 raft_consensus.cc:738] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.306988  1230 consensus_queue.cc:260] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.223737   921 leader_election.cc:304] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:21.313185  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:21.287809   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0f22700fc70d4f14ae51a834900db22b because the tablet was not found. No further retry: Not found: Tablet not found: 0f22700fc70d4f14ae51a834900db22b
W20250628 07:52:21.318132   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet deb5a9ae7ab5463cae35ff1086fd8348 because the tablet was not found. No further retry: Not found: Tablet not found: deb5a9ae7ab5463cae35ff1086fd8348
W20250628 07:52:21.318698   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 67a81eb4342740e5bc16ee7f5eacb489 because the tablet was not found. No further retry: Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:21.320745   805 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.321830   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba2a10acdf0462bb05628351cdbbbe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.322016  1234 consensus_queue.cc:260] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.342104  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 36eebc4a46bd454da5bc176df2a9680f (DEFAULT_TABLE table=default.Table_d74975b9cd2942c98865802e679ef4d5 [id=af0bd34359f84793a28be3c92a52612e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.343151  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36eebc4a46bd454da5bc176df2a9680f. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:21.234668   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 35) took 1381 ms (client timeout 1000 ms). Trace:
W20250628 07:52:21.351063   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 235626us
I20250628 07:52:21.361616  1625 raft_consensus.cc:491] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.362169  1625 raft_consensus.cc:513] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.364037  1625 leader_election.cc:290] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.365206  1230 ts_tablet_manager.cc:1428] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.109s	user 0.006s	sys 0.001s
I20250628 07:52:21.366184  1230 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:21.366241  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:21.366948  1127 raft_consensus.cc:2466] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:21.367767  1640 ts_tablet_manager.cc:1905] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:21.368046   758 leader_election.cc:304] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:21.369022  1625 raft_consensus.cc:2802] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:21.369396  1625 raft_consensus.cc:491] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:21.369751  1625 raft_consensus.cc:3058] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.371021   801 tablet_service.cc:1468] Processing CreateTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.372087   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fa4299a9e0f4b409d04c8c5900901c0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.374423   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:21.374734   964 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (DEFAULT_TABLE table=default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.375708   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba2a10acdf0462bb05628351cdbbbe4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.376647  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 4ba7d2634cb74f6f858a5a3db8493b6a (DEFAULT_TABLE table=default.Table_7c51004dd5724c6499326369c8712f8a [id=a7843107992b478b8e0b560eb2f13fa5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.377601  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4ba7d2634cb74f6f858a5a3db8493b6a. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:21.293366   539 rpcz_store.cc:269] 0628 07:52:18.480358 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:18.495235 (+ 14877us) service_pool.cc:225] Handling call
0628 07:52:18.559628 (+ 64393us) spinlock_profiling.cc:243] Waited 55.1 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:20.483918 (+1924290us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.483942 (+    24us) write_op.cc:454] Released schema lock
0628 07:52:21.218537 (+734595us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.464513 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.464894 (+   381us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.464917 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:20.464930 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:20.465333 (+   403us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.465396 (+    63us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.465419 (+    23us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:20.465660 (+   241us) tablet.cc:699] Row locks acquired
0628 07:52:20.465673 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:20.465792 (+   119us) write_op.cc:270] Start()
0628 07:52:20.465871 (+    79us) write_op.cc:276] Timestamp: P: 1751097140465622 usec, L: 0
0628 07:52:20.465887 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.466470 (+   583us) log.cc:844] Serialized 913 byte log entry
0628 07:52:20.476760 (+ 10290us) op_driver.cc:464] REPLICATION: finished
0628 07:52:20.482171 (+  5411us) write_op.cc:301] APPLY: starting
0628 07:52:20.482227 (+    56us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:20.482323 (+    96us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:20.482404 (+    81us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:20.482650 (+   246us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:20.482795 (+   145us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:20.482835 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:20.482989 (+   154us) log.cc:844] Serialized 29 byte log entry
0628 07:52:20.483109 (+   120us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:20.483173 (+    64us) write_op.cc:454] Released schema lock
0628 07:52:20.483207 (+    34us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"reported_tablets":16,"spinlock_wait_cycles":126662528,"child_traces":[["op",{"apply.queue_time_us":5151,"apply.run_cpu_time_us":1215,"apply.run_wall_time_us":1298,"num_ops":2,"prepare.queue_time_us":115666,"prepare.run_cpu_time_us":2232,"prepare.run_wall_time_us":2297,"raft.queue_time_us":1159,"raft.run_cpu_time_us":712,"raft.run_wall_time_us":713,"replication_time_us":10768}]]}
I20250628 07:52:21.381075  1625 raft_consensus.cc:513] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.383667  1106 tablet_service.cc:1468] Processing CreateTablet for tablet ef060774191146168ee8d1e13c9cd71b (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.384564  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef060774191146168ee8d1e13c9cd71b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.387284   962 tablet_service.cc:1468] Processing CreateTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.388291   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fa4299a9e0f4b409d04c8c5900901c0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.375056   985 raft_consensus.cc:2466] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:21.392271  1623 raft_consensus.cc:491] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.392665  1623 raft_consensus.cc:513] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.395010  1623 leader_election.cc:290] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:21.397946   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:21.399452  1062 leader_election.cc:343] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.400262  1645 tablet.cc:1722] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:21.389446  1504 raft_consensus.cc:2802] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:21.403762   966 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.403297  1504 raft_consensus.cc:491] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20250628 07:52:21.389636   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 171736us
I20250628 07:52:21.404532  1504 raft_consensus.cc:3058] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.409554   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 83f8dcff296a4429a5f16bde59c8ebee
I20250628 07:52:21.411556  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee with delete_type TABLET_DATA_DELETED (replaced by tablet 661d92febf1142f99815f70ae9b27edc) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.412074   545 catalog_manager.cc:6652] ReplaceTablet: tablet 83f8dcff296a4429a5f16bde59c8ebee deleted and replaced by tablet 661d92febf1142f99815f70ae9b27edc
I20250628 07:52:21.412784  1594 ts_tablet_manager.cc:1918] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:21.413185  1594 log.cc:1199] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:21.413972  1594 ts_tablet_manager.cc:1939] T 3223005021ae4e38b4ac65af460d0ad0 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:21.414793  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 106) took 1201 ms. Trace:
I20250628 07:52:21.415113  1594 rpcz_store.cc:276] 0628 07:52:20.213317 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.213429 (+   112us) service_pool.cc:225] Handling call
0628 07:52:20.287863 (+ 74434us) ts_tablet_manager.cc:1145] Deleting tablet 3223005021ae4e38b4ac65af460d0ad0
0628 07:52:20.287901 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:21.072514 (+784613us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:21.412555 (+340041us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:21.414742 (+  2187us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":73968}
W20250628 07:52:21.417822   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 152233us
I20250628 07:52:21.435307   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1642348a16574345b31cebda23575102
I20250628 07:52:21.437585   542 catalog_manager.cc:6652] ReplaceTablet: tablet 1642348a16574345b31cebda23575102 deleted and replaced by tablet cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:21.437654  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 1642348a16574345b31cebda23575102 with delete_type TABLET_DATA_DELETED (replaced by tablet cc0bf0edebb140209531ccf917fd1ed8) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.410686  1642 tablet.cc:1722] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:21.445286   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:21.453408   966 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.454267  1623 raft_consensus.cc:491] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.454691  1623 raft_consensus.cc:513] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.454927  1536 raft_consensus.cc:513] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.455884   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:21.456482  1623 leader_election.cc:290] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:21.456490   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:21.456966   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:21.453409  1599 consensus_queue.cc:1035] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.008s
I20250628 07:52:21.458407  1504 raft_consensus.cc:513] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:21.345296   538 rpcz_store.cc:269] 0628 07:52:19.853478 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:19.929898 (+ 76420us) service_pool.cc:225] Handling call
0628 07:52:19.949869 (+ 19971us) spinlock_profiling.cc:243] Waited 17.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.234607 (+1284738us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:20.720994 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:20.721378 (+   384us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:20.721400 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:20.721413 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:20.721645 (+   232us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:20.721680 (+    35us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:20.721700 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:20.721837 (+   137us) tablet.cc:699] Row locks acquired
0628 07:52:20.721849 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:20.721958 (+   109us) write_op.cc:270] Start()
0628 07:52:20.722040 (+    82us) write_op.cc:276] Timestamp: P: 1751097140721750 usec, L: 0
0628 07:52:20.722056 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:20.722633 (+   577us) log.cc:844] Serialized 533 byte log entry
0628 07:52:21.142496 (+419863us) op_driver.cc:464] REPLICATION: finished
0628 07:52:21.143042 (+   546us) write_op.cc:301] APPLY: starting
0628 07:52:21.143111 (+    69us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:21.143250 (+   139us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:21.143371 (+   121us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:21.143594 (+   223us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:21.143771 (+   177us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:21.143837 (+    66us) write_op.cc:312] APPLY: finished
0628 07:52:21.144053 (+   216us) log.cc:844] Serialized 23 byte log entry
0628 07:52:21.144980 (+   927us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.145034 (+    54us) write_op.cc:454] Released schema lock
0628 07:52:21.145077 (+    43us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:21.145369 (+   292us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.145403 (+    34us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":14,"spinlock_wait_cycles":39505664,"child_traces":[["op",{"apply.queue_time_us":364,"apply.run_cpu_time_us":1529,"apply.run_wall_time_us":2271,"num_ops":1,"prepare.queue_time_us":140,"prepare.run_cpu_time_us":1833,"prepare.run_wall_time_us":3469,"raft.queue_time_us":70,"raft.run_cpu_time_us":15,"raft.run_wall_time_us":15,"replication_time_us":420324}]]}
I20250628 07:52:21.466411  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:21.466938  1127 raft_consensus.cc:3058] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.473624  1106 tablet_service.cc:1468] Processing CreateTablet for tablet cb62cf19aaf04444845358cc8ad611fb (DEFAULT_TABLE table=default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.474547  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cb62cf19aaf04444845358cc8ad611fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.498534   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 16a8bee148664e948529d32c0cf2c793 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.467423  1599 consensus_queue.cc:1035] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.034s
I20250628 07:52:21.467483  1504 leader_election.cc:290] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.503400  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 107) took 1145 ms. Trace:
I20250628 07:52:21.503743  1594 rpcz_store.cc:276] 0628 07:52:20.358074 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.358262 (+   188us) service_pool.cc:225] Handling call
0628 07:52:21.460327 (+1102065us) ts_tablet_manager.cc:1145] Deleting tablet 0f29c8ffb7df410295278eee9c578b03
0628 07:52:21.461518 (+  1191us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:21.503328 (+ 41810us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1084490}
W20250628 07:52:21.504418   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet fab31b073e2e49318b4118851f81b6d2 consensus state has no leader
I20250628 07:52:21.510210  1229 tablet_bootstrap.cc:492] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:21.510623  1229 ts_tablet_manager.cc:1397] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.725s	user 0.010s	sys 0.003s
I20250628 07:52:21.512462   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:21.513360   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_514e2f1c447845de9ac4cfa1f2c7aab0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:21.514057   966 tablet_service.cc:1468] Processing CreateTablet for tablet 36eebc4a46bd454da5bc176df2a9680f (DEFAULT_TABLE table=default.Table_d74975b9cd2942c98865802e679ef4d5 [id=af0bd34359f84793a28be3c92a52612e]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:21.515183   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_514e2f1c447845de9ac4cfa1f2c7aab0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:21.515194  1229 raft_consensus.cc:357] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.515786  1229 raft_consensus.cc:383] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.516032  1229 raft_consensus.cc:738] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.517509   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ead071b51bc040dfb0a5a966a79a1094"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:21.518949   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ead071b51bc040dfb0a5a966a79a1094 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:21.519224  1062 leader_election.cc:343] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.520803  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:21.521384  1129 raft_consensus.cc:3058] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.524988   961 tablet_service.cc:1468] Processing CreateTablet for tablet 4ba7d2634cb74f6f858a5a3db8493b6a (DEFAULT_TABLE table=default.Table_7c51004dd5724c6499326369c8712f8a [id=a7843107992b478b8e0b560eb2f13fa5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.383909   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.384152  1234 ts_tablet_manager.cc:1428] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.087s	user 0.003s	sys 0.003s
I20250628 07:52:21.396668   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:21.458031   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.460755   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.463022   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:21.470295   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 16a8bee148664e948529d32c0cf2c793 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.483317   799 tablet_service.cc:1468] Processing CreateTablet for tablet 36eebc4a46bd454da5bc176df2a9680f (DEFAULT_TABLE table=default.Table_d74975b9cd2942c98865802e679ef4d5 [id=af0bd34359f84793a28be3c92a52612e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.538408   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36eebc4a46bd454da5bc176df2a9680f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.539728  1625 leader_election.cc:290] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.540771   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1eac71e98ae44f41bb17816618c7d897" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:21.541361   823 raft_consensus.cc:3058] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.544384   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8c1491faa1be485792b6193d4ca22f37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:21.544905   825 raft_consensus.cc:3058] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.546118   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:21.546612   984 raft_consensus.cc:3058] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.509542   800 tablet_service.cc:1468] Processing CreateTablet for tablet 4ba7d2634cb74f6f858a5a3db8493b6a (DEFAULT_TABLE table=default.Table_7c51004dd5724c6499326369c8712f8a [id=a7843107992b478b8e0b560eb2f13fa5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.549275  1602 consensus_queue.cc:1035] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:21.549839   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4ba7d2634cb74f6f858a5a3db8493b6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.549940  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "950fea8f6bfb409f8868fe5e88d3dddb" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:21.550467  1126 raft_consensus.cc:3058] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.536809  1234 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:21.553362  1062 leader_election.cc:343] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.553759  1062 leader_election.cc:304] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:21.554378  1623 raft_consensus.cc:2747] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:21.554704  1062 leader_election.cc:343] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.555034  1062 leader_election.cc:304] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:21.555604  1623 raft_consensus.cc:2747] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:21.555871   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:21.556710   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:21.557188  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 109) took 1198 ms. Trace:
I20250628 07:52:21.557516   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:21.557740  1127 raft_consensus.cc:2466] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:21.557582  1594 rpcz_store.cc:276] 0628 07:52:20.358821 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.358959 (+   138us) service_pool.cc:225] Handling call
0628 07:52:21.504589 (+1145630us) ts_tablet_manager.cc:1145] Deleting tablet cb62cf19aaf04444845358cc8ad611fb
0628 07:52:21.513272 (+  8683us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:21.524098 (+ 10826us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1145185}
I20250628 07:52:21.558198   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:21.559394   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:21.566289   825 raft_consensus.cc:2466] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:21.569615  1230 tablet_bootstrap.cc:700] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Time spent opening tablet: real 0.181s	user 0.003s	sys 0.000s
I20250628 07:52:21.570658  1230 tablet_bootstrap.cc:654] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:21.574282   920 leader_election.cc:304] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:21.574491  1244 ts_tablet_manager.cc:1918] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:21.574855  1244 log.cc:1199] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:21.575006  1599 raft_consensus.cc:2802] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:21.575556  1244 ts_tablet_manager.cc:1939] T fa96605dbabe4f0fbc26b057232960a4 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:21.575487  1599 raft_consensus.cc:695] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:21.576203  1599 consensus_queue.cc:237] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.577140   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]) successfully deleted
W20250628 07:52:21.577617   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 64844.8us
W20250628 07:52:21.578209   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0f29c8ffb7df410295278eee9c578b03 already in progress: creating tablet
W20250628 07:52:21.578778   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:21.589696   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36eebc4a46bd454da5bc176df2a9680f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.594625   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4ba7d2634cb74f6f858a5a3db8493b6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.597653  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 108) took 1239 ms. Trace:
I20250628 07:52:21.598251  1594 rpcz_store.cc:276] 0628 07:52:20.358464 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.359021 (+   557us) service_pool.cc:225] Handling call
0628 07:52:21.558518 (+1199497us) ts_tablet_manager.cc:1145] Deleting tablet cb62cf19aaf04444845358cc8ad611fb
0628 07:52:21.597262 (+ 38744us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:21.597566 (+   304us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1198781}
I20250628 07:52:21.599866  1594 tablet_replica.cc:331] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:21.606911  1129 raft_consensus.cc:2466] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:21.619974   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet cc0bf0edebb140209531ccf917fd1ed8 has no consensus state
I20250628 07:52:21.620460   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1eac71e98ae44f41bb17816618c7d897
I20250628 07:52:21.621783   543 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 178) took 1062 ms. Trace:
I20250628 07:52:21.609203  1623 raft_consensus.cc:491] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.627586  1623 raft_consensus.cc:513] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.622085   543 rpcz_store.cc:276] 0628 07:52:20.559331 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.045603 (+486272us) service_pool.cc:225] Handling call
0628 07:52:21.183800 (+138197us) spinlock_profiling.cc:243] Waited 137 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea941046 00007f5be86601a4 00007f5be8660132 00007f5be8660061 00007f5be865ffdc 00007f5be865f292 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.262775 (+ 78975us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:21.264205 (+  1430us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:21.264352 (+   147us) catalog_manager.cc:2799] Locking tablets
0628 07:52:21.264982 (+   630us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:21.438264 (+173282us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:21.441139 (+  2875us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:21.620382 (+179243us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:21.621715 (+  1333us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:21.353330 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:21.353767 (+   437us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:21.353793 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:21.353808 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:21.354131 (+   323us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:21.354170 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:21.354196 (+    26us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:21.354489 (+   293us) tablet.cc:699] Row locks acquired
0628 07:52:21.354504 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:21.354613 (+   109us) write_op.cc:270] Start()
0628 07:52:21.354697 (+    84us) write_op.cc:276] Timestamp: P: 1751097141354535 usec, L: 0
0628 07:52:21.354715 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:21.355224 (+   509us) log.cc:844] Serialized 731 byte log entry
0628 07:52:21.419348 (+ 64124us) op_driver.cc:464] REPLICATION: finished
0628 07:52:21.420052 (+   704us) write_op.cc:301] APPLY: starting
0628 07:52:21.420104 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:21.420208 (+   104us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:21.420292 (+    84us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:21.420559 (+   267us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:21.420693 (+   134us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:21.420741 (+    48us) write_op.cc:312] APPLY: finished
0628 07:52:21.420905 (+   164us) log.cc:844] Serialized 29 byte log entry
0628 07:52:21.423317 (+  2412us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.423377 (+    60us) write_op.cc:454] Released schema lock
0628 07:52:21.423407 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:21.423601 (+   194us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:21.423628 (+    27us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":315555584,"child_traces":[["op",{"apply.queue_time_us":560,"apply.run_cpu_time_us":1271,"apply.run_wall_time_us":3525,"num_ops":2,"prepare.queue_time_us":46548,"prepare.run_cpu_time_us":2101,"prepare.run_wall_time_us":3208,"replication_time_us":64543}]]}
I20250628 07:52:21.629565  1623 leader_election.cc:290] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:21.633235  1594 raft_consensus.cc:2241] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:21.616304  1126 raft_consensus.cc:2466] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:21.626559  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 1eac71e98ae44f41bb17816618c7d897 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:21 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.633864  1234 tablet_bootstrap.cc:654] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:21.634559  1594 raft_consensus.cc:2270] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:21.631381   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:21.662515  1062 leader_election.cc:343] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.516564  1229 consensus_queue.cc:260] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.664522  1640 ts_tablet_manager.cc:1918] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:21.664994  1229 ts_tablet_manager.cc:1428] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.154s	user 0.006s	sys 0.001s
I20250628 07:52:21.470896  1536 leader_election.cc:290] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.633427   921 leader_election.cc:304] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:21.666805  1536 raft_consensus.cc:2802] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:21.622347   986 raft_consensus.cc:2466] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:21.642688   962 tablet_service.cc:1468] Processing CreateTablet for tablet ef060774191146168ee8d1e13c9cd71b (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.644258   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee with delete_type TABLET_DATA_DELETED (replaced by tablet 661d92febf1142f99815f70ae9b27edc) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.667321  1536 raft_consensus.cc:695] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:21.645916   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 1642348a16574345b31cebda23575102 with delete_type TABLET_DATA_DELETED (replaced by tablet cc0bf0edebb140209531ccf917fd1ed8) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.649163   960 tablet_service.cc:1468] Processing CreateTablet for tablet cb62cf19aaf04444845358cc8ad611fb (DEFAULT_TABLE table=default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.668074  1536 consensus_queue.cc:237] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.668922   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cb62cf19aaf04444845358cc8ad611fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.673790   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef060774191146168ee8d1e13c9cd71b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.686561  1594 ts_tablet_manager.cc:1905] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:21.690109  1640 log.cc:1199] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:21.691092  1640 ts_tablet_manager.cc:1939] T d62ecf32bcc54310b9114d5655be7673 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:21.693717   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet d62ecf32bcc54310b9114d5655be7673 (table default.Table_1612b9b2623a4d708b4af562bed58f69 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
I20250628 07:52:21.694195  1640 tablet_replica.cc:331] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:21.694999  1640 raft_consensus.cc:2241] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:21.695448  1640 raft_consensus.cc:2270] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:21.687408  1229 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:21.710304   921 consensus_peers.cc:489] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c84f2570b2da45d0b3ce6428723263a5. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:21.651989   984 raft_consensus.cc:2466] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:21.746497  1229 tablet_bootstrap.cc:654] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:21.762395  1640 ts_tablet_manager.cc:1905] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:21.760654  1536 consensus_queue.cc:1035] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:21.767817  1599 raft_consensus.cc:491] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.773641  1599 raft_consensus.cc:513] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.776955  1599 leader_election.cc:290] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:21.777868  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:21.778785  1127 raft_consensus.cc:2466] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:21.779963   921 leader_election.cc:304] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:21.780804  1599 raft_consensus.cc:2802] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:21.781172  1599 raft_consensus.cc:491] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:21.781484  1599 raft_consensus.cc:3058] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:21.792600   757 leader_election.cc:304] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:21.793831   758 leader_election.cc:304] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:21.793965  1625 raft_consensus.cc:2802] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
W20250628 07:52:21.794490   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 76036.6us
I20250628 07:52:21.794557  1625 raft_consensus.cc:695] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:21.795353  1625 consensus_queue.cc:237] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.797134  1244 tablet_replica.cc:331] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:21.797228  1626 raft_consensus.cc:2802] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:21.797603   805 tablet_service.cc:1468] Processing CreateTablet for tablet ef060774191146168ee8d1e13c9cd71b (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.797968  1244 raft_consensus.cc:2241] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:21.798369  1244 pending_rounds.cc:70] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Trying to abort 1 pending ops.
I20250628 07:52:21.798570  1244 pending_rounds.cc:77] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493883458711552 op_type: NO_OP noop_request { }
I20250628 07:52:21.798857  1244 raft_consensus.cc:2887] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:21.799485  1244 raft_consensus.cc:2270] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:21.802592   823 raft_consensus.cc:2466] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:21.805265  1626 raft_consensus.cc:695] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:21.806001  1626 consensus_queue.cc:237] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.808064   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ef060774191146168ee8d1e13c9cd71b. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:21.816100   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 97630.2us
I20250628 07:52:21.822718  1230 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:21.823115  1230 ts_tablet_manager.cc:1397] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.457s	user 0.011s	sys 0.000s
I20250628 07:52:21.823808  1623 raft_consensus.cc:491] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:21.824254  1623 raft_consensus.cc:513] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.826089  1623 leader_election.cc:290] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:21.826169   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet fa96605dbabe4f0fbc26b057232960a4 (table default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]) successfully deleted
I20250628 07:52:21.827898   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 1eac71e98ae44f41bb17816618c7d897 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:21 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:21.827839   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:21.831336  1062 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.832828  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.833573   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:21.835513   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 38) took 989 ms (client timeout 1000 ms). Trace:
I20250628 07:52:21.841454  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:21.842921  1244 ts_tablet_manager.cc:1905] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:21.844784   541 rpcz_store.cc:269] 0628 07:52:20.845940 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.995354 (+149414us) service_pool.cc:225] Handling call
0628 07:52:21.177851 (+182497us) spinlock_profiling.cc:243] Waited 179 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.835462 (+657611us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":178252,"reported_tablets":20,"spinlock_wait_cycles":411845504}
I20250628 07:52:21.850841   825 raft_consensus.cc:3058] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:21.855139  1062 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.855525  1062 leader_election.cc:304] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:21.856431  1623 raft_consensus.cc:2747] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:21.860303   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee with delete_type TABLET_DATA_DELETED (replaced by tablet 661d92febf1142f99815f70ae9b27edc) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.861163   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:21.861377   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:21.861776   825 raft_consensus.cc:2466] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:21.863301  1062 leader_election.cc:343] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:21.863734  1062 leader_election.cc:304] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:21.863974   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 1642348a16574345b31cebda23575102 with delete_type TABLET_DATA_DELETED (replaced by tablet cc0bf0edebb140209531ccf917fd1ed8) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.864702  1623 raft_consensus.cc:2747] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:21.866366   800 tablet_service.cc:1468] Processing CreateTablet for tablet cb62cf19aaf04444845358cc8ad611fb (DEFAULT_TABLE table=default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:21.867399   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cb62cf19aaf04444845358cc8ad611fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:21.875111  1230 raft_consensus.cc:357] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.877779   799 tablet_service.cc:1515] Processing DeleteTablet for tablet 1eac71e98ae44f41bb17816618c7d897 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:21 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:21.879822  1234 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:21.880323  1234 ts_tablet_manager.cc:1397] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.344s	user 0.006s	sys 0.006s
I20250628 07:52:21.882498  1600 consensus_queue.cc:1035] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:21.882462  1234 raft_consensus.cc:357] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.883067  1234 raft_consensus.cc:383] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.883296  1234 raft_consensus.cc:738] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.883872  1234 consensus_queue.cc:260] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.884176  1600 consensus_queue.cc:1035] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:21.831722   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 37) took 987 ms (client timeout 1000 ms). Trace:
W20250628 07:52:21.876856   533 catalog_manager.cc:5001] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): alter failed for tablet 3223005021ae4e38b4ac65af460d0ad0 (table default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]),no further retry: Not found: Tablet not found: 3223005021ae4e38b4ac65af460d0ad0
W20250628 07:52:21.888810   546 rpcz_store.cc:269] 0628 07:52:20.843884 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.991854 (+147970us) service_pool.cc:225] Handling call
0628 07:52:21.175056 (+183202us) spinlock_profiling.cc:243] Waited 167 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:21.831664 (+656608us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":175148,"reported_tablets":14,"spinlock_wait_cycles":384940800}
I20250628 07:52:21.890367   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_11e96390464141b9a308e3f956e1a84d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:21.891947   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_11e96390464141b9a308e3f956e1a84d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:21.893425   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_35424882587f46d1be44e2d8984956b7" } modify_external_catalogs: true
I20250628 07:52:21.893682   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b86a1653daf740d9a4927425c711c0b2" } new_table_name: "default.Table_58cbee214d3d4a7ab7452f436eb56db3" modify_external_catalogs: true
W20250628 07:52:21.894515   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:21.899852   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6ce1e375ffee4e2db52c58c171ba4d08"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:21.902976   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6ce1e375ffee4e2db52c58c171ba4d08 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:21.921010  1230 raft_consensus.cc:383] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:21.921603  1230 raft_consensus.cc:738] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:21.922928  1230 consensus_queue.cc:260] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.933079  1599 raft_consensus.cc:513] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:21.934705  1244 ts_tablet_manager.cc:1918] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:21.953390  1244 log.cc:1199] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:21.954571  1244 ts_tablet_manager.cc:1939] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:21.956125  1640 ts_tablet_manager.cc:1918] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:21.963088  1244 tablet_replica.cc:331] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:21.964326  1244 raft_consensus.cc:2241] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:21.970399  1244 raft_consensus.cc:2270] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:21.978564  1640 log.cc:1199] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/16a8bee148664e948529d32c0cf2c793
I20250628 07:52:21.979360  1640 ts_tablet_manager.cc:1939] T 16a8bee148664e948529d32c0cf2c793 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
W20250628 07:52:21.948724   920 consensus_peers.cc:489] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:21.997542  1640 tablet_replica.cc:331] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:22.000700  1640 raft_consensus.cc:2241] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:22.002086  1640 raft_consensus.cc:2270] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.003201  1665 raft_consensus.cc:491] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 82f43cdfab544b17a6a71033983e65d4)
I20250628 07:52:22.003671  1665 raft_consensus.cc:513] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.005553  1640 ts_tablet_manager.cc:1905] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.007778   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:22.017148  1665 leader_election.cc:290] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:22.017268  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5e1b0dbbc21b445d8115d963f6aba2a1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:22.019874   758 leader_election.cc:343] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:22.021742  1601 consensus_queue.cc:1035] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:21.964668  1229 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:21.968082  1645 tablet.cc:1722] T 872ebf0393fa40cf82feebaad1ca3fa3 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:22.025492  1229 ts_tablet_manager.cc:1397] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.338s	user 0.007s	sys 0.002s
I20250628 07:52:22.039186   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:22.039824   823 raft_consensus.cc:3058] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:22.045224  1663 raft_consensus.cc:491] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:22.026067   758 leader_election.cc:304] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:22.045698  1663 raft_consensus.cc:513] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.048259   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d62ecf32bcc54310b9114d5655be7673" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:22.049415  1663 leader_election.cc:290] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:22.049659   757 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:22.049932  1663 raft_consensus.cc:2747] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:21.932751   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_35424882587f46d1be44e2d8984956b7" } modify_external_catalogs: true
W20250628 07:52:21.985533   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet cc0bf0edebb140209531ccf917fd1ed8 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet cc0bf0edebb140209531ccf917fd1ed8 has no consensus state
I20250628 07:52:21.990657   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 16a8bee148664e948529d32c0cf2c793 (table default.Table_0adee4e1249840be952b0a99dba4c161 [id=f59d42b137cf4536a002585b728017f6]) successfully deleted
W20250628 07:52:22.052229   533 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 61627.2us
I20250628 07:52:22.054315  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4e9d585d5f0c4469828561a7df4fb917" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:22.054865  1127 raft_consensus.cc:3058] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:22.057977  1599 leader_election.cc:290] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:22.059770   758 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:22.060190   758 leader_election.cc:304] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:22.022192  1234 ts_tablet_manager.cc:1428] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.142s	user 0.004s	sys 0.000s
I20250628 07:52:22.061933  1234 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:22.063880  1665 raft_consensus.cc:2747] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:22.064620  1230 ts_tablet_manager.cc:1428] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.241s	user 0.008s	sys 0.000s
I20250628 07:52:22.065589  1230 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:22.012531   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 532ba24d0d644313b9ef4e3ec10b0fed consensus state has no leader
I20250628 07:52:22.070472  1536 consensus_queue.cc:1035] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:22.089217  1676 tablet.cc:1722] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:22.089722  1678 tablet.cc:1722] T 872ebf0393fa40cf82feebaad1ca3fa3 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:22.091351  1244 ts_tablet_manager.cc:1905] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.100347   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (table default.Table_6d90b18a9f474219933cb8e8511d01d6 [id=4d5813f4497543018828699e64223907]) successfully deleted
I20250628 07:52:22.032593  1229 raft_consensus.cc:357] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.100878  1229 raft_consensus.cc:383] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.101213  1229 raft_consensus.cc:738] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.101830  1229 consensus_queue.cc:260] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.095216  1640 ts_tablet_manager.cc:1918] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:22.106191  1640 log.cc:1199] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/5e1b0dbbc21b445d8115d963f6aba2a1
I20250628 07:52:22.106966  1640 ts_tablet_manager.cc:1939] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
W20250628 07:52:22.118564   920 consensus_peers.cc:489] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:22.101547   800 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.124729  1640 tablet_replica.cc:331] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:22.102416   799 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.103181   798 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.125516  1640 raft_consensus.cc:2241] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:22.129993  1640 raft_consensus.cc:2270] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.087744   823 raft_consensus.cc:2466] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:22.134101   920 leader_election.cc:304] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:22.135193  1601 raft_consensus.cc:2802] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:22.135732  1640 ts_tablet_manager.cc:1905] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.135675  1601 raft_consensus.cc:695] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:22.136560  1601 consensus_queue.cc:237] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.123612   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (table default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]) successfully deleted
W20250628 07:52:22.145445   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 45120.1us
I20250628 07:52:22.147992   541 master_service.cc:948] ReplaceTablet: received request to replace tablet 482095693aac476faf997ea8e1a4b460 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:22.149294   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1935e83524ad453c8175710332a3c459" } modify_external_catalogs: true
I20250628 07:52:22.149947   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1935e83524ad453c8175710332a3c459" } modify_external_catalogs: true
I20250628 07:52:22.152976  1229 ts_tablet_manager.cc:1428] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.122s	user 0.004s	sys 0.000s
I20250628 07:52:22.154304  1229 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:22.161159  1234 tablet_bootstrap.cc:654] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:22.169647   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 71516us
I20250628 07:52:22.171001   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f5b1b57d6e754cccb6eb0a1405289fde"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.173238   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f5b1b57d6e754cccb6eb0a1405289fde in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.188143  1594 ts_tablet_manager.cc:1918] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:22.188647  1594 log.cc:1199] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/6e912d18dcff4a938a867ea06ee7e0f0
I20250628 07:52:22.193681  1594 ts_tablet_manager.cc:1939] T 6e912d18dcff4a938a867ea06ee7e0f0 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:22.195420  1230 tablet_bootstrap.cc:654] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.198216   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2e20136b218e40b6a54ab31e306e147e" } new_table_name: "default.Table_7924c3ed1cf34beba7ba362d781f34de" modify_external_catalogs: true
I20250628 07:52:22.184664  1127 raft_consensus.cc:2466] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:22.212208   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bd2bbc34f4c04ac3993ac60498df657f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.214201   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bd2bbc34f4c04ac3993ac60498df657f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.214811   545 master_service.cc:948] ReplaceTablet: received request to replace tablet 16a8bee148664e948529d32c0cf2c793 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:22.217644  1229 tablet_bootstrap.cc:654] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.227396   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111), 
I20250628 07:52:22.228024   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:22.235008  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 111) took 1841 ms. Trace:
I20250628 07:52:22.235452  1594 rpcz_store.cc:276] 0628 07:52:20.393199 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.393313 (+   114us) service_pool.cc:225] Handling call
0628 07:52:21.599642 (+1206329us) ts_tablet_manager.cc:1145] Deleting tablet 6e912d18dcff4a938a867ea06ee7e0f0
0628 07:52:21.599701 (+    59us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:21.948973 (+349272us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.187884 (+238911us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.234946 (+ 47062us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1205765}
I20250628 07:52:22.241489  1234 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:22.241892  1234 ts_tablet_manager.cc:1397] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.180s	user 0.008s	sys 0.001s
I20250628 07:52:22.243988  1234 raft_consensus.cc:357] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.244580  1234 raft_consensus.cc:383] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.244827  1234 raft_consensus.cc:738] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.251156   805 tablet_service.cc:1468] Processing CreateTablet for tablet 7eca25db66fa42a798fc002d43f7e3c3 (DEFAULT_TABLE table=default.Table_2af9142e039c4b78a79c8f8252e66c86 [id=be8d8530baf34f4ea04d59598d765502]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.252308   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7eca25db66fa42a798fc002d43f7e3c3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.254942  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 7eca25db66fa42a798fc002d43f7e3c3 (DEFAULT_TABLE table=default.Table_2af9142e039c4b78a79c8f8252e66c86 [id=be8d8530baf34f4ea04d59598d765502]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.255873  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7eca25db66fa42a798fc002d43f7e3c3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.259390  1640 ts_tablet_manager.cc:1918] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:22.259789  1640 log.cc:1199] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/83f8dcff296a4429a5f16bde59c8ebee
I20250628 07:52:22.260491  1640 ts_tablet_manager.cc:1939] T 83f8dcff296a4429a5f16bde59c8ebee P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:22.262842  1640 tablet_replica.cc:331] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:22.263574  1640 raft_consensus.cc:2241] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:22.264050  1640 raft_consensus.cc:2270] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.267118  1234 consensus_queue.cc:260] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.268528   960 tablet_service.cc:1468] Processing CreateTablet for tablet 7eca25db66fa42a798fc002d43f7e3c3 (DEFAULT_TABLE table=default.Table_2af9142e039c4b78a79c8f8252e66c86 [id=be8d8530baf34f4ea04d59598d765502]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.268875   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 83f8dcff296a4429a5f16bde59c8ebee (table default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]) successfully deleted
I20250628 07:52:22.270464   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7eca25db66fa42a798fc002d43f7e3c3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.273921  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 5475a89692eb450d82cde521ec8c9a73 (DEFAULT_TABLE table=default.Table_1f0b83a168f54608ae33309446b7b2aa [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.274879  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5475a89692eb450d82cde521ec8c9a73. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.281144  1601 raft_consensus.cc:491] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:22.281713  1601 raft_consensus.cc:513] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.284431  1234 ts_tablet_manager.cc:1428] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.042s	user 0.005s	sys 0.000s
I20250628 07:52:22.286108   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "16a8bee148664e948529d32c0cf2c793" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:22.287382   920 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:22.287890  1601 leader_election.cc:290] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:22.289278   921 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 16a8bee148664e948529d32c0cf2c793
I20250628 07:52:22.289733   921 leader_election.cc:304] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:22.291091  1601 raft_consensus.cc:2747] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:22.296201  1244 ts_tablet_manager.cc:1918] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:22.296540  1244 log.cc:1199] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:22.297256  1244 ts_tablet_manager.cc:1939] T 2b51d6076e344c8f8e388b77b4b23928 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:22.295320  1234 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:22.300901   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 6e912d18dcff4a938a867ea06ee7e0f0 (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:22.302850   798 tablet_service.cc:1468] Processing CreateTablet for tablet 5475a89692eb450d82cde521ec8c9a73 (DEFAULT_TABLE table=default.Table_1f0b83a168f54608ae33309446b7b2aa [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.303916   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5475a89692eb450d82cde521ec8c9a73. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.303680   961 tablet_service.cc:1468] Processing CreateTablet for tablet 5475a89692eb450d82cde521ec8c9a73 (DEFAULT_TABLE table=default.Table_1f0b83a168f54608ae33309446b7b2aa [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.304690   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5475a89692eb450d82cde521ec8c9a73. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.305619  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 116) took 1579 ms. Trace:
I20250628 07:52:22.305963  1594 rpcz_store.cc:276] 0628 07:52:20.726477 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.727370 (+   893us) service_pool.cc:225] Handling call
0628 07:52:22.305344 (+1577974us) ts_tablet_manager.cc:1145] Deleting tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17
0628 07:52:22.305391 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.305559 (+   168us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1577259}
I20250628 07:52:22.308528  1594 tablet_replica.cc:331] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:22.309373  1594 raft_consensus.cc:2241] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:22.310009  1594 raft_consensus.cc:2270] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.316202   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d2097dcb437041b98aef0c014b4e34e9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:22.316471   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c" } new_table_name: "default.Table_1feb9beed180403ab066bd5685be58f3" modify_external_catalogs: true
W20250628 07:52:22.317790   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d2097dcb437041b98aef0c014b4e34e9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.321238  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 5ec8c1d9386a4922b55df5e06039858d (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.322223  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ec8c1d9386a4922b55df5e06039858d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.322240   799 tablet_service.cc:1468] Processing CreateTablet for tablet 5ec8c1d9386a4922b55df5e06039858d (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.323774   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ec8c1d9386a4922b55df5e06039858d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.324512  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 9579dd44e5e241aea2671be28e16ef7c (DEFAULT_TABLE table=default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.325505  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9579dd44e5e241aea2671be28e16ef7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.326845  1594 ts_tablet_manager.cc:1905] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.314975  1229 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:22.329360  1229 ts_tablet_manager.cc:1397] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.175s	user 0.011s	sys 0.000s
I20250628 07:52:22.329576  1108 tablet_service.cc:1468] Processing CreateTablet for tablet febd000d51e2480e81766b292304b4f5 (DEFAULT_TABLE table=default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c [id=99e9926911304df09ecd802e344295af]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.330476  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet febd000d51e2480e81766b292304b4f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.331563  1229 raft_consensus.cc:357] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.332170  1229 raft_consensus.cc:383] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.332412  1229 raft_consensus.cc:738] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.338337   962 tablet_service.cc:1468] Processing CreateTablet for tablet 5ec8c1d9386a4922b55df5e06039858d (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.339233   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ec8c1d9386a4922b55df5e06039858d. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:22.349382   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet c4a5d0deb0a049a2b6cc2ac5a8dd3c17 because the tablet was not found. No further retry: Not found: Tablet not found: c4a5d0deb0a049a2b6cc2ac5a8dd3c17
I20250628 07:52:22.351290   800 tablet_service.cc:1468] Processing CreateTablet for tablet 9579dd44e5e241aea2671be28e16ef7c (DEFAULT_TABLE table=default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.351723   966 tablet_service.cc:1468] Processing CreateTablet for tablet 9579dd44e5e241aea2671be28e16ef7c (DEFAULT_TABLE table=default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.352362   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9579dd44e5e241aea2671be28e16ef7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.352592   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9579dd44e5e241aea2671be28e16ef7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.353165   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:22.354302   538 catalog_manager.cc:5582] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: HEALTHY } } }
I20250628 07:52:22.355427   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:22.356427   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:22.357244   538 catalog_manager.cc:5582] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:22.358306   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:22.359050   538 catalog_manager.cc:5582] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:22.332993  1229 consensus_queue.cc:260] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.362826  1640 ts_tablet_manager.cc:1905] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.365206  1229 ts_tablet_manager.cc:1428] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.036s	user 0.006s	sys 0.000s
I20250628 07:52:22.366348  1229 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:22.382817  1623 raft_consensus.cc:491] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:22.383453  1623 raft_consensus.cc:513] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.385537  1623 leader_election.cc:290] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:22.386471   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "01feb8ef40dc4707bd6251fc2d87f46f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:22.387653  1062 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:22.388783   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "01feb8ef40dc4707bd6251fc2d87f46f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:22.390655   801 tablet_service.cc:1468] Processing CreateTablet for tablet febd000d51e2480e81766b292304b4f5 (DEFAULT_TABLE table=default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c [id=99e9926911304df09ecd802e344295af]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.391557   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet febd000d51e2480e81766b292304b4f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.391346   964 tablet_service.cc:1468] Processing CreateTablet for tablet febd000d51e2480e81766b292304b4f5 (DEFAULT_TABLE table=default.Table_b0d2daf26a0c44e4aae0f6587a1a4a2c [id=99e9926911304df09ecd802e344295af]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.392218   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet febd000d51e2480e81766b292304b4f5. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:22.393711  1062 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:22.394058  1062 leader_election.cc:304] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:22.394950  1623 raft_consensus.cc:2747] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:22.420133  1594 ts_tablet_manager.cc:1918] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:22.420514  1594 log.cc:1199] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:22.421629  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 46e0aa090f544e8aa0a426c61925f345 (DEFAULT_TABLE table=default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c [id=680e55a59eba4e39b0d2def0036257e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.422715  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46e0aa090f544e8aa0a426c61925f345. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.424705  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 (DEFAULT_TABLE table=default.Table_0979356e8b1a441fa942c0359545cfc9 [id=e518ccb92cf248d09e0f974067eab1bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.425153   802 tablet_service.cc:1468] Processing CreateTablet for tablet 132cd82d39e84e85a37f21ae208fd165 (DEFAULT_TABLE table=default.Table_7017192359304fa08eb3df34abc8c536 [id=5699b079b64a4864a611eb6bb02ed256]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.425968  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e2ecd14e8524f018dd3c0aecd564218. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.426067   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 132cd82d39e84e85a37f21ae208fd165. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.445322  1594 ts_tablet_manager.cc:1939] T 2b51d6076e344c8f8e388b77b4b23928 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:22.445842   965 tablet_service.cc:1468] Processing CreateTablet for tablet 46e0aa090f544e8aa0a426c61925f345 (DEFAULT_TABLE table=default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c [id=680e55a59eba4e39b0d2def0036257e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.446887   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46e0aa090f544e8aa0a426c61925f345. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.447391  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 118) took 1702 ms. Trace:
I20250628 07:52:22.447726  1594 rpcz_store.cc:276] 0628 07:52:20.745154 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.745453 (+   299us) service_pool.cc:225] Handling call
0628 07:52:22.308386 (+1562933us) ts_tablet_manager.cc:1145] Deleting tablet 2b51d6076e344c8f8e388b77b4b23928
0628 07:52:22.308426 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.386122 (+ 77696us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.419912 (+ 33790us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.447335 (+ 27423us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1562330}
I20250628 07:52:22.449039   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 2b51d6076e344c8f8e388b77b4b23928 (table default.Table_c22b7ab2180346e093151880dbd4cb83 [id=3602ce86ce23434d83a8641b502843c4]) successfully deleted
I20250628 07:52:22.449932   805 tablet_service.cc:1468] Processing CreateTablet for tablet 46e0aa090f544e8aa0a426c61925f345 (DEFAULT_TABLE table=default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c [id=680e55a59eba4e39b0d2def0036257e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.451442   963 tablet_service.cc:1468] Processing CreateTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 (DEFAULT_TABLE table=default.Table_0979356e8b1a441fa942c0359545cfc9 [id=e518ccb92cf248d09e0f974067eab1bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.452389   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e2ecd14e8524f018dd3c0aecd564218. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.452540  1229 tablet_bootstrap.cc:654] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.455093   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46e0aa090f544e8aa0a426c61925f345. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.458912  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 117) took 1714 ms. Trace:
I20250628 07:52:22.459239  1594 rpcz_store.cc:276] 0628 07:52:20.744179 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.744286 (+   107us) service_pool.cc:225] Handling call
0628 07:52:22.458641 (+1714355us) ts_tablet_manager.cc:1145] Deleting tablet 474cec4459954e10a18de031946a2346
0628 07:52:22.458693 (+    52us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.458852 (+   159us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1706481}
W20250628 07:52:22.460377   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 474cec4459954e10a18de031946a2346 because the tablet was not found. No further retry: Not found: Tablet not found: 474cec4459954e10a18de031946a2346
I20250628 07:52:22.460563  1600 consensus_queue.cc:1035] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:22.477540  1594 tablet_replica.cc:331] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:22.478487  1594 raft_consensus.cc:2241] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:22.479456  1594 raft_consensus.cc:2270] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.481854  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 132cd82d39e84e85a37f21ae208fd165 (DEFAULT_TABLE table=default.Table_7017192359304fa08eb3df34abc8c536 [id=5699b079b64a4864a611eb6bb02ed256]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.482908  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 132cd82d39e84e85a37f21ae208fd165. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.484731   962 tablet_service.cc:1468] Processing CreateTablet for tablet 132cd82d39e84e85a37f21ae208fd165 (DEFAULT_TABLE table=default.Table_7017192359304fa08eb3df34abc8c536 [id=5699b079b64a4864a611eb6bb02ed256]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.484897  1101 tablet_service.cc:1468] Processing CreateTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 (DEFAULT_TABLE table=default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.485594   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 132cd82d39e84e85a37f21ae208fd165. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.485970  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2d49e5a06384758bce5e7328b6a10f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.488134   799 tablet_service.cc:1468] Processing CreateTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 (DEFAULT_TABLE table=default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.489993  1230 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:22.490352  1230 ts_tablet_manager.cc:1397] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.425s	user 0.013s	sys 0.005s
I20250628 07:52:22.490674   801 tablet_service.cc:1468] Processing CreateTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 (DEFAULT_TABLE table=default.Table_0979356e8b1a441fa942c0359545cfc9 [id=e518ccb92cf248d09e0f974067eab1bd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.492419   961 tablet_service.cc:1468] Processing CreateTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 (DEFAULT_TABLE table=default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.492522  1230 raft_consensus.cc:357] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.494706  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 137) took 1037 ms. Trace:
I20250628 07:52:22.495087  1244 rpcz_store.cc:276] 0628 07:52:21.457770 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.457908 (+   138us) service_pool.cc:225] Handling call
0628 07:52:21.962960 (+505052us) ts_tablet_manager.cc:1145] Deleting tablet 2b51d6076e344c8f8e388b77b4b23928
0628 07:52:21.962996 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.161072 (+198076us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.294415 (+133343us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:22.494649 (+200234us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":426098}
I20250628 07:52:22.496325  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 136) took 1038 ms. Trace:
I20250628 07:52:22.498871  1601 consensus_queue.cc:1035] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:22.502453   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 2b51d6076e344c8f8e388b77b4b23928 (table default.Table_c22b7ab2180346e093151880dbd4cb83 [id=3602ce86ce23434d83a8641b502843c4]) successfully deleted
I20250628 07:52:22.509312   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet e4e02676d24a4345908dbbc42fc9c1ab
I20250628 07:52:22.517397   960 tablet_service.cc:1515] Processing DeleteTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:22.525920   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:22.528398  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:22.532253   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 482095693aac476faf997ea8e1a4b460
I20250628 07:52:22.533429   541 catalog_manager.cc:6652] ReplaceTablet: tablet 482095693aac476faf997ea8e1a4b460 deleted and replaced by tablet af13b56df157443c994f747e52b0f458
I20250628 07:52:22.535109  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet 482095693aac476faf997ea8e1a4b460 with delete_type TABLET_DATA_DELETED (replaced by tablet af13b56df157443c994f747e52b0f458) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:22.537246   964 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:22.530705  1594 ts_tablet_manager.cc:1905] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.540297  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:22.542196  1230 raft_consensus.cc:383] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.542503  1230 raft_consensus.cc:738] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.542855  1229 tablet_bootstrap.cc:492] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:22.543216  1229 ts_tablet_manager.cc:1397] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.177s	user 0.007s	sys 0.003s
I20250628 07:52:22.543105  1230 consensus_queue.cc:260] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.547343   960 tablet_service.cc:1515] Processing DeleteTablet for tablet 482095693aac476faf997ea8e1a4b460 with delete_type TABLET_DATA_DELETED (replaced by tablet af13b56df157443c994f747e52b0f458) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:22.548122   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet febd000d51e2480e81766b292304b4f5 consensus state has no leader
I20250628 07:52:22.493826   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2d49e5a06384758bce5e7328b6a10f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.521476   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2d49e5a06384758bce5e7328b6a10f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.531047   803 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.555979   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e2ecd14e8524f018dd3c0aecd564218. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.548199   802 tablet_service.cc:1515] Processing DeleteTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.553979   798 tablet_service.cc:1468] Processing CreateTablet for tablet 58e7ec1f221f41ca8f6b019faeec3377 (DEFAULT_TABLE table=default.Table_fd173dac22494e51a7adca1ec4817a45 [id=9a2c30b2d6a6409dbd3565a4b74ba210]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.542212   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 482095693aac476faf997ea8e1a4b460 with delete_type TABLET_DATA_DELETED (replaced by tablet af13b56df157443c994f747e52b0f458) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:22.560448   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58e7ec1f221f41ca8f6b019faeec3377. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.560910  1229 raft_consensus.cc:357] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.561589  1229 raft_consensus.cc:383] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.561825  1229 raft_consensus.cc:738] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.555505  1244 rpcz_store.cc:276] 0628 07:52:21.457491 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.460638 (+  3147us) service_pool.cc:225] Handling call
0628 07:52:22.496116 (+1035478us) ts_tablet_manager.cc:1145] Deleting tablet 3223005021ae4e38b4ac65af460d0ad0
0628 07:52:22.496154 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.496283 (+   129us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":959117}
I20250628 07:52:22.569759   540 master_service.cc:948] ReplaceTablet: received request to replace tablet d62ecf32bcc54310b9114d5655be7673 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:22.571646   964 tablet_service.cc:1468] Processing CreateTablet for tablet 58e7ec1f221f41ca8f6b019faeec3377 (DEFAULT_TABLE table=default.Table_fd173dac22494e51a7adca1ec4817a45 [id=9a2c30b2d6a6409dbd3565a4b74ba210]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.572804   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58e7ec1f221f41ca8f6b019faeec3377. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:22.574175   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 3223005021ae4e38b4ac65af460d0ad0 because the tablet was not found. No further retry: Not found: Tablet not found: 3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:22.575186  1230 ts_tablet_manager.cc:1428] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.085s	user 0.003s	sys 0.003s
I20250628 07:52:22.576668  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 138) took 1106 ms. Trace:
I20250628 07:52:22.576951  1244 rpcz_store.cc:276] 0628 07:52:21.470031 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.470144 (+   113us) service_pool.cc:225] Handling call
0628 07:52:22.576348 (+1106204us) ts_tablet_manager.cc:1145] Deleting tablet 16a8bee148664e948529d32c0cf2c793
0628 07:52:22.576388 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.576616 (+   228us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1038803}
I20250628 07:52:22.582360  1244 tablet_replica.cc:331] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
W20250628 07:52:22.582377   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 16a8bee148664e948529d32c0cf2c793 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 16a8bee148664e948529d32c0cf2c793 already in progress: creating tablet
I20250628 07:52:22.584233  1244 raft_consensus.cc:2241] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:22.570828  1229 consensus_queue.cc:260] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.602877  1229 ts_tablet_manager.cc:1428] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.059s	user 0.007s	sys 0.000s
I20250628 07:52:22.612389  1229 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:22.613715  1244 raft_consensus.cc:2270] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:22.616199  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 58e7ec1f221f41ca8f6b019faeec3377 (DEFAULT_TABLE table=default.Table_fd173dac22494e51a7adca1ec4817a45 [id=9a2c30b2d6a6409dbd3565a4b74ba210]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.617908   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1d25d1c32a764b60a6265908bd978817"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.619429   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1d25d1c32a764b60a6265908bd978817 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.587538  1234 tablet_bootstrap.cc:654] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.621336   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3f52d53f779c43b38b6cd8d7332cd51f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.622757   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3f52d53f779c43b38b6cd8d7332cd51f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.636562   799 tablet_service.cc:1468] Processing CreateTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 (DEFAULT_TABLE table=default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.637769   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc0bf0edebb140209531ccf917fd1ed8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.640496   965 tablet_service.cc:1468] Processing CreateTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 (DEFAULT_TABLE table=default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.641587   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc0bf0edebb140209531ccf917fd1ed8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.658277  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58e7ec1f221f41ca8f6b019faeec3377. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.675788   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:22.676844   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:22.678130   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:22.679242   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:22.681556  1230 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:22.684309  1234 tablet_bootstrap.cc:492] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:22.684751  1234 ts_tablet_manager.cc:1397] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.390s	user 0.009s	sys 0.000s
I20250628 07:52:22.693363  1244 ts_tablet_manager.cc:1905] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:22.702068  1103 tablet_service.cc:1468] Processing CreateTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 (DEFAULT_TABLE table=default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.703307  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc0bf0edebb140209531ccf917fd1ed8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.701988   805 tablet_service.cc:1468] Processing CreateTablet for tablet 661d92febf1142f99815f70ae9b27edc (DEFAULT_TABLE table=default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.715221   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 661d92febf1142f99815f70ae9b27edc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.736653   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_bd2bbc34f4c04ac3993ac60498df657f" } modify_external_catalogs: true
I20250628 07:52:22.737277   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_bd2bbc34f4c04ac3993ac60498df657f" } modify_external_catalogs: true
I20250628 07:52:22.736756   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_96f7aea7a2b445139de2bf323b687f73"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.739876   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_96f7aea7a2b445139de2bf323b687f73 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.739508   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_09cc8edc019c4d26936ec96d72adbb0d" } modify_external_catalogs: true
I20250628 07:52:22.740471   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_09cc8edc019c4d26936ec96d72adbb0d" } modify_external_catalogs: true
I20250628 07:52:22.763074   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1612b9b2623a4d708b4af562bed58f69" } new_table_name: "default.Table_8dd91f20a94d4b7fa01cb1a8b96a48e0" modify_external_catalogs: true
I20250628 07:52:22.783936   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:22.784917   800 tablet_service.cc:1468] Processing CreateTablet for tablet 0dffb3eb778946869b914e5b05a67a69 (DEFAULT_TABLE table=default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.789402  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 661d92febf1142f99815f70ae9b27edc (DEFAULT_TABLE table=default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.791785  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 661d92febf1142f99815f70ae9b27edc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.816507  1234 raft_consensus.cc:357] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.817305  1234 raft_consensus.cc:383] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.817582  1234 raft_consensus.cc:738] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.818248  1234 consensus_queue.cc:260] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.827075  1229 tablet_bootstrap.cc:654] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.841238   960 tablet_service.cc:1468] Processing CreateTablet for tablet 661d92febf1142f99815f70ae9b27edc (DEFAULT_TABLE table=default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.842342   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 661d92febf1142f99815f70ae9b27edc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.842691   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:22.844081   539 catalog_manager.cc:5582] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
W20250628 07:52:22.848518   921 consensus_peers.cc:489] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5e1b0dbbc21b445d8115d963f6aba2a1. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:22.849366  1600 consensus_queue.cc:1035] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:22.866541   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_de48ef8f76e3475a8ec0919c675b8711"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:22.868245   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_de48ef8f76e3475a8ec0919c675b8711 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:22.879766   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]). Expected version 2 got 0
I20250628 07:52:22.880726   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:22.881278   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:22.907223   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dffb3eb778946869b914e5b05a67a69. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.911973  1229 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:22.912384  1229 ts_tablet_manager.cc:1397] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.300s	user 0.019s	sys 0.000s
I20250628 07:52:22.917275   545 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet fe6525adc6b041c9b15a71c539b93ebe
W20250628 07:52:22.918151   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8c1491faa1be485792b6193d4ca22f37 consensus state has no leader
I20250628 07:52:22.922824  1229 raft_consensus.cc:357] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.923421  1229 raft_consensus.cc:383] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:22.923673  1229 raft_consensus.cc:738] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:22.924187  1229 consensus_queue.cc:260] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:22.924878   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 36) took 1085 ms (client timeout 1000 ms). Trace:
W20250628 07:52:22.929207   538 rpcz_store.cc:269] 0628 07:52:21.839940 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.840056 (+   116us) service_pool.cc:225] Handling call
0628 07:52:22.924803 (+1084747us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:22.619293 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:22.619663 (+   370us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:22.619684 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:22.619698 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:22.620078 (+   380us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:22.620114 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:22.620137 (+    23us) tablet.cc:683] Acquiring locks for 3 operations
0628 07:52:22.620372 (+   235us) tablet.cc:699] Row locks acquired
0628 07:52:22.620385 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:22.620482 (+    97us) write_op.cc:270] Start()
0628 07:52:22.620556 (+    74us) write_op.cc:276] Timestamp: P: 1751097142620435 usec, L: 0
0628 07:52:22.620570 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:22.643946 (+ 23376us) log.cc:844] Serialized 1293 byte log entry
0628 07:52:22.661317 (+ 17371us) op_driver.cc:464] REPLICATION: finished
0628 07:52:22.661636 (+   319us) write_op.cc:301] APPLY: starting
0628 07:52:22.661685 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:22.661800 (+   115us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:22.661965 (+   165us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:22.662279 (+   314us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:22.662420 (+   141us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=3
0628 07:52:22.662463 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:22.662677 (+   214us) log.cc:844] Serialized 35 byte log entry
0628 07:52:22.662812 (+   135us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:22.662885 (+    73us) write_op.cc:454] Released schema lock
0628 07:52:22.662915 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:22.663205 (+   290us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:22.663228 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":18,"child_traces":[["op",{"apply.queue_time_us":170,"apply.run_cpu_time_us":1547,"apply.run_wall_time_us":1546,"num_ops":3,"prepare.queue_time_us":84174,"prepare.run_cpu_time_us":2552,"prepare.run_wall_time_us":32214,"replication_time_us":40657}]]}
I20250628 07:52:22.955538  1229 ts_tablet_manager.cc:1428] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.043s	user 0.006s	sys 0.000s
I20250628 07:52:22.959177  1229 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:22.969956  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 0dffb3eb778946869b914e5b05a67a69 (DEFAULT_TABLE table=default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.976570  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dffb3eb778946869b914e5b05a67a69. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.983896  1106 tablet_service.cc:1468] Processing CreateTablet for tablet bed1ea07e3d44c08adf37ced9dbef85e (DEFAULT_TABLE table=default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:22.985163  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed1ea07e3d44c08adf37ced9dbef85e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:22.993749  1230 tablet_bootstrap.cc:654] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:22.996129   963 tablet_service.cc:1468] Processing CreateTablet for tablet 0dffb3eb778946869b914e5b05a67a69 (DEFAULT_TABLE table=default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.004853  1700 raft_consensus.cc:491] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.005568  1700 raft_consensus.cc:513] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.007584  1700 leader_election.cc:290] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.008850   959 tablet_service.cc:1515] Processing DeleteTablet for tablet 3223005021ae4e38b4ac65af460d0ad0 with delete_type TABLET_DATA_DELETED (replaced by tablet bed1ea07e3d44c08adf37ced9dbef85e) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.008944   966 tablet_service.cc:1468] Processing CreateTablet for tablet bed1ea07e3d44c08adf37ced9dbef85e (DEFAULT_TABLE table=default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.009879   959 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.010073  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:23.010684  1127 raft_consensus.cc:2466] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.011090   959 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.011793   921 leader_election.cc:304] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:23.016052  1600 raft_consensus.cc:2802] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.016351  1600 raft_consensus.cc:491] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.016639  1600 raft_consensus.cc:3058] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.040868  1230 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:23.041270  1230 ts_tablet_manager.cc:1397] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.360s	user 0.008s	sys 0.001s
I20250628 07:52:23.000737  1244 ts_tablet_manager.cc:1918] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:23.043432  1244 log.cc:1199] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/83f8dcff296a4429a5f16bde59c8ebee
I20250628 07:52:23.043843  1230 raft_consensus.cc:357] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.044822   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed1ea07e3d44c08adf37ced9dbef85e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.045168  1244 ts_tablet_manager.cc:1939] T 83f8dcff296a4429a5f16bde59c8ebee P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:23.052861   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dffb3eb778946869b914e5b05a67a69. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.060223  1698 raft_consensus.cc:491] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.060635  1698 raft_consensus.cc:513] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.061769  1234 ts_tablet_manager.cc:1428] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.377s	user 0.009s	sys 0.000s
I20250628 07:52:23.062383  1698 leader_election.cc:290] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.064395  1229 tablet_bootstrap.cc:654] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.069152  1640 ts_tablet_manager.cc:1918] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:23.069531  1640 log.cc:1199] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1642348a16574345b31cebda23575102
I20250628 07:52:23.070262  1640 ts_tablet_manager.cc:1939] T 1642348a16574345b31cebda23575102 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:23.076983  1234 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:23.078039  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 142) took 1218 ms. Trace:
I20250628 07:52:23.079438  1244 rpcz_store.cc:276] 0628 07:52:21.860070 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.860172 (+   102us) service_pool.cc:225] Handling call
0628 07:52:22.582218 (+722046us) ts_tablet_manager.cc:1145] Deleting tablet 83f8dcff296a4429a5f16bde59c8ebee
0628 07:52:22.582254 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.927781 (+345527us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.000493 (+ 72712us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.077982 (+ 77489us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":27301,"tablet-delete.queue_time_us":721389}
I20250628 07:52:23.081542  1244 tablet_replica.cc:331] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:23.082828  1244 raft_consensus.cc:2241] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:23.087373  1600 raft_consensus.cc:513] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.089915  1600 leader_election.cc:290] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.091988  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:23.092669  1128 raft_consensus.cc:3058] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.093814  1244 raft_consensus.cc:2270] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.097057  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:23.104969  1244 ts_tablet_manager.cc:1905] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.044469  1230 raft_consensus.cc:383] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.106014  1230 raft_consensus.cc:738] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:23.106683  1230 consensus_queue.cc:260] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.110723  1230 ts_tablet_manager.cc:1428] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.069s	user 0.005s	sys 0.001s
W20250628 07:52:23.106927   758 consensus_peers.cc:489] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:23.116451  1663 consensus_queue.cc:1035] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:23.124997   922 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 111259us
I20250628 07:52:23.131160  1717 tablet.cc:1722] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.134048  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 145) took 1696 ms. Trace:
I20250628 07:52:23.136759  1230 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:22.954185   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20250628 07:52:23.141490   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 187353us
W20250628 07:52:22.963115   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 9ba2a10acdf0462bb05628351cdbbbe4 consensus state has no leader
I20250628 07:52:23.143013   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_514e2f1c447845de9ac4cfa1f2c7aab0" } modify_external_catalogs: true
I20250628 07:52:23.143479   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_514e2f1c447845de9ac4cfa1f2c7aab0" } modify_external_catalogs: true
I20250628 07:52:22.969554   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6c95fbef63ae4a34aff4518c2d8cbdad"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.144824   962 tablet_service.cc:1468] Processing CreateTablet for tablet 8feee3be1a664cae9da417b190163a94 (DEFAULT_TABLE table=default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.145861   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8feee3be1a664cae9da417b190163a94. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:23.146287   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6c95fbef63ae4a34aff4518c2d8cbdad in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:22.989800   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_8dd91f20a94d4b7fa01cb1a8b96a48e0 [id=a9599116e3c04051a9484736a07d350f], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 9ba2a10acdf0462bb05628351cdbbbe4 consensus state has no leader
I20250628 07:52:22.995983   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_019f48b4d4734b4ba9d765410d4ebe1d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.151026  1623 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.151435  1623 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.151925  1106 tablet_service.cc:1468] Processing CreateTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (DEFAULT_TABLE table=default.Table_055d241428174f7891615d2422293f39 [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.144961   964 tablet_service.cc:1468] Processing CreateTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (DEFAULT_TABLE table=default.Table_055d241428174f7891615d2422293f39 [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.152948  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4fcefaa9e954b52bbbee6e0a2f2f939. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.153255   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4fcefaa9e954b52bbbee6e0a2f2f939. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.153775  1623 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:23.134441  1640 rpcz_store.cc:276] 0628 07:52:21.437398 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.437512 (+   114us) service_pool.cc:225] Handling call
0628 07:52:22.262696 (+825184us) ts_tablet_manager.cc:1145] Deleting tablet 1642348a16574345b31cebda23575102
0628 07:52:22.262738 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.607985 (+345247us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.068888 (+460903us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.133987 (+ 65099us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":824535}
I20250628 07:52:23.154874   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1642348a16574345b31cebda23575102 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
I20250628 07:52:23.157383  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 83f8dcff296a4429a5f16bde59c8ebee with delete_type TABLET_DATA_DELETED (replaced by tablet 661d92febf1142f99815f70ae9b27edc) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:23.159276   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_019f48b4d4734b4ba9d765410d4ebe1d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:23.161129  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.163357  1640 tablet_replica.cc:331] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:23.164299  1640 raft_consensus.cc:2241] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.164875  1640 raft_consensus.cc:2270] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.148401   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:23.166445  1700 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.166846  1700 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.168859  1599 raft_consensus.cc:491] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.169186  1700 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.169299  1599 raft_consensus.cc:513] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.170325  1229 tablet_bootstrap.cc:492] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:23.170725  1229 ts_tablet_manager.cc:1397] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.212s	user 0.005s	sys 0.004s
I20250628 07:52:23.172647  1230 tablet_bootstrap.cc:654] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.175954  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:23.176445  1126 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.193084  1128 raft_consensus.cc:2466] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:23.200742  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 1642348a16574345b31cebda23575102 with delete_type TABLET_DATA_DELETED (replaced by tablet cc0bf0edebb140209531ccf917fd1ed8) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.202425  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 16a8bee148664e948529d32c0cf2c793 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.202626  1720 tablet.cc:1722] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.204646   544 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 1a6c6f102c6b46dd880a448d659ae84f
I20250628 07:52:23.206758  1230 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:23.207121  1230 ts_tablet_manager.cc:1397] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.072s	user 0.008s	sys 0.000s
I20250628 07:52:23.209867  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 8feee3be1a664cae9da417b190163a94 (DEFAULT_TABLE table=default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.210835  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8feee3be1a664cae9da417b190163a94. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.221616  1230 raft_consensus.cc:357] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.222257  1230 raft_consensus.cc:383] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.222487  1230 raft_consensus.cc:738] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:23.226866  1108 consensus_meta.cc:287] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.502s	user 0.009s	sys 0.000s
I20250628 07:52:23.233768  1640 ts_tablet_manager.cc:1905] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.236580   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:23.238543  1062 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.241050  1234 tablet_bootstrap.cc:654] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.242899   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1feb9beed180403ab066bd5685be58f3" } new_table_name: "default.Table_be426ff6be794d35845ebfa8e3357e07" modify_external_catalogs: true
I20250628 07:52:23.246218  1719 tablet.cc:1722] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.246587   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_85406b18e2c04407b80adfc0e5ee9587"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.230393  1230 consensus_queue.cc:260] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:23.051944   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 38) took 998 ms (client timeout 1000 ms). Trace:
I20250628 07:52:23.075160   540 master_service.cc:948] ReplaceTablet: received request to replace tablet 1642348a16574345b31cebda23575102 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:23.089214   547 master_service.cc:948] ReplaceTablet: received request to replace tablet 5ec8c1d9386a4922b55df5e06039858d from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:23.231003   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 114772us
I20250628 07:52:23.263805  1713 consensus_queue.cc:1035] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:23.264977  1663 raft_consensus.cc:491] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.084656   546 catalog_manager.cc:5582] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:23.265395   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:23.265404  1663 raft_consensus.cc:513] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.265933   985 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:23.266220   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:23.267051  1663 leader_election.cc:290] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.267737   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:23.268666   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:23.269547   546 catalog_manager.cc:5582] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:23.271317   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8c1491faa1be485792b6193d4ca22f37 (table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a]). Expected version 1 got 0
I20250628 07:52:23.271209  1229 raft_consensus.cc:357] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:22.937434   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:23.271842  1229 raft_consensus.cc:383] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.272073  1229 raft_consensus.cc:738] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:23.139066   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_78e2ac50470645f49e4cadc30ff61a59"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.171020  1599 leader_election.cc:290] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.276327   921 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:23.277079   921 leader_election.cc:304] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:23.166445   984 raft_consensus.cc:2391] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:23.277923  1244 ts_tablet_manager.cc:1918] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:23.278297  1244 log.cc:1199] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/1642348a16574345b31cebda23575102
I20250628 07:52:23.278962  1244 ts_tablet_manager.cc:1939] T 1642348a16574345b31cebda23575102 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:23.280148  1599 raft_consensus.cc:2802] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.280454  1599 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.280787  1599 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.286124  1701 raft_consensus.cc:2802] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:23.286634  1701 raft_consensus.cc:695] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
W20250628 07:52:23.287323   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_78e2ac50470645f49e4cadc30ff61a59 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:23.287840   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_85406b18e2c04407b80adfc0e5ee9587 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:23.287559  1701 consensus_queue.cc:237] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.272583  1229 consensus_queue.cc:260] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.297698  1062 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:23.305358   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 39) took 1359 ms (client timeout 1000 ms). Trace:
I20250628 07:52:23.315961   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:23.316818   543 rpcz_store.cc:269] 0628 07:52:21.945883 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.946000 (+   117us) service_pool.cc:225] Handling call
0628 07:52:23.305291 (+1359291us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":21}
I20250628 07:52:23.300475  1663 consensus_queue.cc:1035] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:23.342610  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 143) took 1478 ms. Trace:
I20250628 07:52:23.343089  1244 rpcz_store.cc:276] 0628 07:52:21.863728 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.863833 (+   105us) service_pool.cc:225] Handling call
0628 07:52:23.081395 (+1217562us) ts_tablet_manager.cc:1145] Deleting tablet 1642348a16574345b31cebda23575102
0628 07:52:23.081435 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.197624 (+116189us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.277698 (+ 80074us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.342518 (+ 64820us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":2250,"tablet-delete.queue_time_us":1216942}
I20250628 07:52:23.348693  1230 ts_tablet_manager.cc:1428] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.141s	user 0.007s	sys 0.000s
I20250628 07:52:23.348802  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:23.349511  1126 raft_consensus.cc:2466] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.350281  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:23.350822  1128 raft_consensus.cc:2466] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:23.351739  1623 raft_consensus.cc:2802] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.352142  1623 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.352447  1623 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.356657  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.377254  1244 tablet_replica.cc:331] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:23.378813  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f (DEFAULT_TABLE table=default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.380311  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83ffa65248fc4db3aecc5f91b648b83f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.386375  1623 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.391661  1244 raft_consensus.cc:2241] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.392258  1244 raft_consensus.cc:2270] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.393887  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.397843  1623 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:23.398334  1623 raft_consensus.cc:491] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.398694  1623 raft_consensus.cc:513] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.402323  1244 ts_tablet_manager.cc:1905] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.319105   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:23.414536   984 raft_consensus.cc:2466] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:23.319278   963 tablet_service.cc:1468] Processing CreateTablet for tablet bfc303c4588d44428984d8203fc05185 (DEFAULT_TABLE table=default.Table_e1df74fb3dfa455bb3300ad20e7a2268 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.416134   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bfc303c4588d44428984d8203fc05185. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.349961   960 tablet_service.cc:1468] Processing CreateTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f (DEFAULT_TABLE table=default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.364153  1594 ts_tablet_manager.cc:1918] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:23.423148  1594 log.cc:1199] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:23.423902  1594 ts_tablet_manager.cc:1939] T fa96605dbabe4f0fbc26b057232960a4 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:23.425879  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 119) took 2679 ms. Trace:
I20250628 07:52:23.426160  1594 rpcz_store.cc:276] 0628 07:52:20.746693 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.752571 (+  5878us) service_pool.cc:225] Handling call
0628 07:52:22.477363 (+1724792us) ts_tablet_manager.cc:1145] Deleting tablet fa96605dbabe4f0fbc26b057232960a4
0628 07:52:22.477416 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:22.932453 (+455037us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.329453 (+397000us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.425825 (+ 96372us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1724309}
I20250628 07:52:23.427270  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 120) took 2674 ms. Trace:
I20250628 07:52:23.427544  1594 rpcz_store.cc:276] 0628 07:52:20.753327 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.753412 (+    85us) service_pool.cc:225] Handling call
0628 07:52:23.427041 (+2673629us) ts_tablet_manager.cc:1145] Deleting tablet 67a81eb4342740e5bc16ee7f5eacb489
0628 07:52:23.427079 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.427222 (+   143us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2673114}
I20250628 07:52:23.428465  1594 tablet_replica.cc:331] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:23.430269   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83ffa65248fc4db3aecc5f91b648b83f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.371817  1599 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.435159  1599 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.443723  1594 raft_consensus.cc:2241] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:23.444635  1594 raft_consensus.cc:2270] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.446564  1594 ts_tablet_manager.cc:1905] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.449966  1623 leader_election.cc:290] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:23.390264  1229 ts_tablet_manager.cc:1428] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.219s	user 0.005s	sys 0.005s
I20250628 07:52:23.455533  1229 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:23.318643   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 5ec8c1d9386a4922b55df5e06039858d
I20250628 07:52:23.458797  1108 tablet_service.cc:1468] Processing CreateTablet for tablet bfc303c4588d44428984d8203fc05185 (DEFAULT_TABLE table=default.Table_e1df74fb3dfa455bb3300ad20e7a2268 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.459165   547 catalog_manager.cc:6652] ReplaceTablet: tablet 5ec8c1d9386a4922b55df5e06039858d deleted and replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307
I20250628 07:52:23.459766  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bfc303c4588d44428984d8203fc05185. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:23.275171   539 rpcz_store.cc:269] 0628 07:52:22.052974 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.053119 (+   145us) service_pool.cc:225] Handling call
0628 07:52:22.060891 (+  7772us) spinlock_profiling.cc:243] Waited 3.28 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:23.051651 (+990760us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:22.996995 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:22.997358 (+   363us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:22.997380 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:22.997392 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:22.997634 (+   242us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:22.997671 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:22.997691 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:22.997831 (+   140us) tablet.cc:699] Row locks acquired
0628 07:52:22.997843 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:22.998060 (+   217us) write_op.cc:270] Start()
0628 07:52:22.998138 (+    78us) write_op.cc:276] Timestamp: P: 1751097142998017 usec, L: 0
0628 07:52:22.998153 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:22.998574 (+   421us) log.cc:844] Serialized 533 byte log entry
0628 07:52:23.006561 (+  7987us) op_driver.cc:464] REPLICATION: finished
0628 07:52:23.011642 (+  5081us) write_op.cc:301] APPLY: starting
0628 07:52:23.011692 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:23.011778 (+    86us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:23.011891 (+   113us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:23.012040 (+   149us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:23.012178 (+   138us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:23.012217 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:23.012374 (+   157us) log.cc:844] Serialized 23 byte log entry
0628 07:52:23.012496 (+   122us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:23.012552 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:23.012582 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:23.042939 (+ 30357us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:23.042964 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":18,"spinlock_wait_cycles":7537408,"child_traces":[["op",{"apply.queue_time_us":4922,"apply.run_cpu_time_us":1124,"apply.run_wall_time_us":1124,"num_ops":1,"prepare.queue_time_us":76942,"prepare.run_cpu_time_us":1783,"prepare.run_wall_time_us":45921,"raft.queue_time_us":802,"raft.run_cpu_time_us":564,"raft.run_wall_time_us":564,"replication_time_us":8318}]]}
I20250628 07:52:23.345566   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2f8967092ad94cb9a5b4b41f44684f30" } new_table_name: "default.Table_4ba728e60a9b472aa1944ff3caf15795" modify_external_catalogs: true
I20250628 07:52:23.463763  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.465610   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_408f187fe62742609a155742ab328fa6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:23.467352   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_408f187fe62742609a155742ab328fa6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:23.372753   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet febd000d51e2480e81766b292304b4f5 consensus state has no leader
I20250628 07:52:23.431963   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a148c40376f64b4fb79113972fe5693f" } modify_external_catalogs: true
I20250628 07:52:23.470052   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a148c40376f64b4fb79113972fe5693f" } modify_external_catalogs: true
I20250628 07:52:23.475387   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:23.476522   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:23.481276   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fa96605dbabe4f0fbc26b057232960a4 (table default.Table_929c4c35bfbb404cafec33d8710f7a1a [id=0259415eb2564b2a966399371ce244fa]) successfully deleted
I20250628 07:52:23.492480   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_cce67d1f9bbb4434a6e80abe70297d1c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.492638   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_915f3db7b34449c4b8124af06f772bc8" } modify_external_catalogs: true
I20250628 07:52:23.493880   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8c1491faa1be485792b6193d4ca22f37 (table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a]). Expected version 1 got 0
I20250628 07:52:23.493824   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_915f3db7b34449c4b8124af06f772bc8" } modify_external_catalogs: true
W20250628 07:52:23.495108   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_cce67d1f9bbb4434a6e80abe70297d1c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:23.495110   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:23.522289   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:23.531432  1109 tablet_service.cc:1468] Processing CreateTablet for tablet b51b51eb951d45a78bd2bffe47834b95 (DEFAULT_TABLE table=default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.532253  1731 raft_consensus.cc:491] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:23.532394   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 67a81eb4342740e5bc16ee7f5eacb489 because the tablet was not found. No further retry: Not found: Tablet not found: 67a81eb4342740e5bc16ee7f5eacb489
I20250628 07:52:23.532527  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b51b51eb951d45a78bd2bffe47834b95. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.374018  1719 tablet.cc:1722] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.532685  1731 raft_consensus.cc:513] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:23.488142   961 consensus_meta.cc:287] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.546s	user 0.008s	sys 0.004s
I20250628 07:52:23.543215   546 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc] alter complete (version 2)
I20250628 07:52:23.543601  1230 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:23.546440  1731 leader_election.cc:290] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:23.549175  1230 tablet_bootstrap.cc:654] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.562242  1230 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:23.562644  1230 ts_tablet_manager.cc:1397] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.019s	user 0.011s	sys 0.000s
I20250628 07:52:23.299896   757 leader_election.cc:304] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:23.581548   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 281523us
I20250628 07:52:23.583864   758 leader_election.cc:304] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:23.583981   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fab31b073e2e49318b4118851f81b6d2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:23.586256  1062 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.586545   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:23.587291   824 raft_consensus.cc:2466] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.588708   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:23.589502   822 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:23.590940  1062 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.591038   823 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e4e02676d24a4345908dbbc42fc9c1ab" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:23.592548   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "482095693aac476faf997ea8e1a4b460" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:23.593153   817 raft_consensus.cc:3058] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:23.608424  1062 leader_election.cc:343] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.564824  1230 raft_consensus.cc:357] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.621587  1230 raft_consensus.cc:383] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.621906  1230 raft_consensus.cc:738] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:23.622738  1230 consensus_queue.cc:260] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.624776  1230 ts_tablet_manager.cc:1428] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.062s	user 0.005s	sys 0.000s
I20250628 07:52:23.597743   816 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:23.636180  1244 ts_tablet_manager.cc:1918] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:23.636808  1244 log.cc:1199] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/1eac71e98ae44f41bb17816618c7d897
W20250628 07:52:23.502213   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 224394us
I20250628 07:52:23.656476  1244 ts_tablet_manager.cc:1939] T 1eac71e98ae44f41bb17816618c7d897 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:23.659720  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 145) took 1782 ms. Trace:
I20250628 07:52:23.659760  1734 raft_consensus.cc:491] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:23.660154  1734 raft_consensus.cc:513] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.660050  1244 rpcz_store.cc:276] 0628 07:52:21.877528 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.877629 (+   101us) service_pool.cc:225] Handling call
0628 07:52:23.377076 (+1499447us) ts_tablet_manager.cc:1145] Deleting tablet 1eac71e98ae44f41bb17816618c7d897
0628 07:52:23.377141 (+    65us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.444240 (+ 67099us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.635677 (+191437us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.659638 (+ 23961us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1498853}
I20250628 07:52:23.660656  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:23.662974  1128 raft_consensus.cc:2391] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:23.663928   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.664431   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 2b51d6076e344c8f8e388b77b4b23928 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.664759   802 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.665482  1734 leader_election.cc:290] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.666011   805 tablet_service.cc:1468] Processing CreateTablet for tablet 8feee3be1a664cae9da417b190163a94 (DEFAULT_TABLE table=default.Table_2f8967092ad94cb9a5b4b41f44684f30 [id=45f854bb205c451392a6f9976a68fd89]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.665916   798 tablet_service.cc:1468] Processing CreateTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f (DEFAULT_TABLE table=default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.666596   801 tablet_service.cc:1468] Processing CreateTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (DEFAULT_TABLE table=default.Table_055d241428174f7891615d2422293f39 [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.667428   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8feee3be1a664cae9da417b190163a94. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.667538   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.668042   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4fcefaa9e954b52bbbee6e0a2f2f939. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.668107   966 tablet_service.cc:1515] Processing DeleteTablet for tablet fa96605dbabe4f0fbc26b057232960a4 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.668534   965 tablet_service.cc:1468] Processing CreateTablet for tablet b51b51eb951d45a78bd2bffe47834b95 (DEFAULT_TABLE table=default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.668656   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:15 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.589516   824 raft_consensus.cc:2466] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.669473   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b51b51eb951d45a78bd2bffe47834b95. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.669970  1663 raft_consensus.cc:2802] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.670009   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.671422   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fab31b073e2e49318b4118851f81b6d2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:23.672399   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:23.673389   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e4e02676d24a4345908dbbc42fc9c1ab" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:23.674499   984 raft_consensus.cc:2391] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 6a58514e01ea4989809bc2992fb2b806 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
W20250628 07:52:23.678449   920 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:23.685123  1594 ts_tablet_manager.cc:1918] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:23.686343  1594 log.cc:1199] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/7a3e80af50b542ce94ea3cb2b2cef3b8
I20250628 07:52:23.689991  1594 ts_tablet_manager.cc:1939] T 7a3e80af50b542ce94ea3cb2b2cef3b8 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:23.656785   961 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:35848 (request call id 146) took 1174 ms. Trace:
I20250628 07:52:23.693378   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83ffa65248fc4db3aecc5f91b648b83f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.693181   961 rpcz_store.cc:276] 0628 07:52:22.482819 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.491715 (+  8896us) service_pool.cc:225] Handling call
0628 07:52:22.493132 (+  1417us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:22.493211 (+    79us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:22.941227 (+448016us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.643418 (+702191us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:23.656729 (+ 13311us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:23.638486  1229 tablet_bootstrap.cc:700] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Time spent opening tablet: real 0.126s	user 0.003s	sys 0.000s
I20250628 07:52:23.694602  1229 tablet_bootstrap.cc:654] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.695986   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:23.701587   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.702877   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.701892  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 123) took 2813 ms. Trace:
I20250628 07:52:23.703306   921 leader_election.cc:304] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:23.703955   800 tablet_service.cc:1468] Processing CreateTablet for tablet bfc303c4588d44428984d8203fc05185 (DEFAULT_TABLE table=default.Table_e1df74fb3dfa455bb3300ad20e7a2268 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.704483  1734 raft_consensus.cc:2802] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.704834  1734 raft_consensus.cc:491] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.704996   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bfc303c4588d44428984d8203fc05185. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.705196  1734 raft_consensus.cc:3058] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.703606  1594 rpcz_store.cc:276] 0628 07:52:20.888695 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:20.888800 (+   105us) service_pool.cc:225] Handling call
0628 07:52:23.428326 (+2539526us) ts_tablet_manager.cc:1145] Deleting tablet 7a3e80af50b542ce94ea3cb2b2cef3b8
0628 07:52:23.428363 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.653227 (+224864us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.684145 (+ 30918us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.701749 (+ 17604us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2527286}
I20250628 07:52:23.709309   803 tablet_service.cc:1468] Processing CreateTablet for tablet b51b51eb951d45a78bd2bffe47834b95 (DEFAULT_TABLE table=default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.709246  1601 consensus_queue.cc:1035] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:23.709578   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 83f8dcff296a4429a5f16bde59c8ebee (table default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]) successfully deleted
I20250628 07:52:23.710325   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b51b51eb951d45a78bd2bffe47834b95. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.711427  1700 consensus_queue.cc:1035] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:23.712432  1734 raft_consensus.cc:513] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:23.715262   920 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.716364  1733 consensus_queue.cc:1035] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:23.716789  1244 tablet_replica.cc:331] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:23.717717  1244 raft_consensus.cc:2241] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:23.718776  1244 pending_rounds.cc:70] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Trying to abort 1 pending ops.
I20250628 07:52:23.719069  1244 pending_rounds.cc:77] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493892822147072 op_type: NO_OP noop_request { }
I20250628 07:52:23.719530  1244 raft_consensus.cc:2887] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:23.719863  1244 raft_consensus.cc:2270] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:23.720384   922 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.0.107.189:33023, user_credentials={real_user=slave}} blocked reactor thread for 46811.6us
W20250628 07:52:23.724758   920 leader_election.cc:343] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.714509  1734 leader_election.cc:290] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.738526   921 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:23.731671  1663 raft_consensus.cc:491] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.741428  1663 raft_consensus.cc:3058] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.742805  1734 raft_consensus.cc:2747] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:23.736055   817 raft_consensus.cc:2466] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:23.749449  1713 raft_consensus.cc:2747] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:23.754093  1244 ts_tablet_manager.cc:1905] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.740849   799 tablet_service.cc:1468] Processing CreateTablet for tablet bed1ea07e3d44c08adf37ced9dbef85e (DEFAULT_TABLE table=default.Table_b89abfdc7dcf4f8db19baf38a00d51ef [id=85eb6f03ddb44b4c99fa174735d121d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:23.757867   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed1ea07e3d44c08adf37ced9dbef85e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:23.761291  1594 tablet_replica.cc:331] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:23.735198  1640 ts_tablet_manager.cc:1918] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
W20250628 07:52:23.722091  1062 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.764846  1640 log.cc:1199] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1eac71e98ae44f41bb17816618c7d897
I20250628 07:52:23.765067  1062 leader_election.cc:304] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:23.765730  1640 ts_tablet_manager.cc:1939] T 1eac71e98ae44f41bb17816618c7d897 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:23.693130  1230 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:23.765861  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 43802.6us
W20250628 07:52:23.766517  1062 leader_election.cc:343] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:23.766834  1062 leader_election.cc:304] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:23.767769  1062 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:23.771850  1732 raft_consensus.cc:2747] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:23.772357  1732 raft_consensus.cc:2747] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:23.772836  1732 raft_consensus.cc:2747] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
W20250628 07:52:23.779826   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 124420us
I20250628 07:52:23.775988  1594 raft_consensus.cc:2241] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:23.786123  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 147) took 2160 ms. Trace:
I20250628 07:52:23.786466  1640 rpcz_store.cc:276] 0628 07:52:21.626262 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.626412 (+   150us) service_pool.cc:225] Handling call
0628 07:52:23.163151 (+1536739us) ts_tablet_manager.cc:1145] Deleting tablet 1eac71e98ae44f41bb17816618c7d897
0628 07:52:23.163213 (+    62us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.692057 (+528844us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.734955 (+ 42898us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.786032 (+ 51077us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":17505,"tablet-delete.queue_time_us":1528949}
I20250628 07:52:23.787606  1640 tablet_replica.cc:331] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:23.787659   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1eac71e98ae44f41bb17816618c7d897 (table default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]) successfully deleted
W20250628 07:52:23.786238   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 124978us
I20250628 07:52:23.788470  1640 raft_consensus.cc:2241] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.788924  1640 raft_consensus.cc:2270] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.789744   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d62ecf32bcc54310b9114d5655be7673" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:23.790324   821 raft_consensus.cc:2466] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:23.790973  1640 ts_tablet_manager.cc:1905] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.748858  1663 raft_consensus.cc:513] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.791998  1594 raft_consensus.cc:2270] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.796295  1663 leader_election.cc:290] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.797431  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:23.807348  1594 ts_tablet_manager.cc:1905] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.809046   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bda937abf44b485aa9fd46ab195442f8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:23.810565   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bda937abf44b485aa9fd46ab195442f8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:23.813095   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ead071b51bc040dfb0a5a966a79a1094" } new_table_name: "default.Table_76b6c7d9f5574825bcc78b251dac8721" modify_external_catalogs: true
W20250628 07:52:23.815726   921 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d62ecf32bcc54310b9114d5655be7673
W20250628 07:52:23.820290   921 leader_election.cc:343] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:23.822968  1229 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:23.823851  1229 ts_tablet_manager.cc:1397] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.368s	user 0.042s	sys 0.003s
I20250628 07:52:23.830124  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:23.835157   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:23.835846   984 raft_consensus.cc:2391] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:23.842324   920 leader_election.cc:304] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:23.843269  1733 raft_consensus.cc:2802] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:23.843571  1733 raft_consensus.cc:491] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:23.843851  1733 raft_consensus.cc:3058] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.850358  1733 raft_consensus.cc:513] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.852403  1733 leader_election.cc:290] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:23.859844  1230 tablet_bootstrap.cc:654] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:23.864809  1719 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:35848 (request call id 151) took 1324 ms. Trace:
I20250628 07:52:23.865235  1719 rpcz_store.cc:276] 0628 07:52:22.539990 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.540673 (+   683us) service_pool.cc:225] Handling call
0628 07:52:23.864743 (+1324070us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:22.588508 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
0628 07:52:22.589293 (+   785us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
0628 07:52:22.589328 (+    35us) alter_schema_op.cc:66] Acquired schema lock
0628 07:52:22.589392 (+    64us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
0628 07:52:22.589528 (+   136us) alter_schema_op.cc:128] START. Timestamp: 1751097142589417
0628 07:52:22.589549 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:22.590125 (+   576us) log.cc:844] Serialized 173 byte log entry
0628 07:52:23.169732 (+579607us) op_driver.cc:464] REPLICATION: finished
0628 07:52:23.373856 (+204124us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
0628 07:52:23.652718 (+278862us) log.cc:844] Serialized 5 byte log entry
0628 07:52:23.808269 (+155551us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.863994 (+ 55725us) log.cc:844] Serialized 14 byte log entry
0628 07:52:23.864603 (+   609us) alter_schema_op.cc:73] Released schema lock
0628 07:52:23.864633 (+    30us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
Metrics: {"child_traces":[["op",{"apply.queue_time_us":203747,"prepare.queue_time_us":16933,"prepare.run_cpu_time_us":18679,"prepare.run_wall_time_us":276499,"raft.queue_time_us":504,"raft.run_cpu_time_us":1679,"raft.run_wall_time_us":1678,"replication_time_us":580053,"thread_start_us":345859,"threads_started":2,"wal-append.queue_time_us":267337}]]}
W20250628 07:52:23.869001   920 consensus_peers.cc:489] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1eac71e98ae44f41bb17816618c7d897. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:23.870427   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 49603.6us
I20250628 07:52:23.872275  1229 raft_consensus.cc:357] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.872982  1229 raft_consensus.cc:383] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.712658   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 1642348a16574345b31cebda23575102 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
W20250628 07:52:23.881336   541 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8feee3be1a664cae9da417b190163a94 consensus state has no leader
W20250628 07:52:23.881431   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 168774us
I20250628 07:52:23.882596   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:23.883224   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 (table default.Table_6d90b18a9f474219933cb8e8511d01d6 [id=4d5813f4497543018828699e64223907]) successfully deleted
I20250628 07:52:23.885479  1229 raft_consensus.cc:738] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
W20250628 07:52:23.885751   921 leader_election.cc:343] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:23.886083  1229 consensus_queue.cc:260] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.884850   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 1eac71e98ae44f41bb17816618c7d897 (table default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]) successfully deleted
I20250628 07:52:23.803941  1747 tablet.cc:1722] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.889986   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_116bc2985c3e44a1962cca8c38c0d338"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:23.890527   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:23.891417   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_116bc2985c3e44a1962cca8c38c0d338 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:23.832875   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29acadb819ef46318e2ba5ba2d3c3c37" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:23.859330   758 leader_election.cc:343] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:23.860157  1234 tablet_bootstrap.cc:492] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:23.861979  1748 tablet.cc:1722] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.900688  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.895007   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.895654   817 raft_consensus.cc:2391] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
I20250628 07:52:23.915242   920 leader_election.cc:304] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:23.916452  1734 raft_consensus.cc:2747] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:23.918624   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet b51b51eb951d45a78bd2bffe47834b95
I20250628 07:52:23.920915  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.922511   965 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:23.926023  1594 ts_tablet_manager.cc:1918] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:23.926335  1229 ts_tablet_manager.cc:1428] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.102s	user 0.005s	sys 0.000s
I20250628 07:52:23.927289  1594 log.cc:1199] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:23.928013  1594 ts_tablet_manager.cc:1939] T c84f2570b2da45d0b3ce6428723263a5 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:23.928567  1229 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:23.930044  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 127) took 2540 ms. Trace:
I20250628 07:52:23.930377  1594 rpcz_store.cc:276] 0628 07:52:21.389539 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.391102 (+  1563us) service_pool.cc:225] Handling call
0628 07:52:23.707671 (+2316569us) ts_tablet_manager.cc:1145] Deleting tablet c84f2570b2da45d0b3ce6428723263a5
0628 07:52:23.707821 (+   150us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.913446 (+205625us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.925791 (+ 12345us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.929993 (+  4202us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":10420,"tablet-delete.queue_time_us":2303422}
I20250628 07:52:23.932577  1230 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:23.932576   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet c84f2570b2da45d0b3ce6428723263a5 (table default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]) successfully deleted
I20250628 07:52:23.933380  1594 tablet_replica.cc:331] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:23.933912  1230 ts_tablet_manager.cc:1397] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.283s	user 0.004s	sys 0.006s
I20250628 07:52:23.934100  1594 raft_consensus.cc:2241] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.934494  1594 pending_rounds.cc:70] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:23.934710  1594 pending_rounds.cc:77] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493892822147072 op_type: NO_OP noop_request { }
I20250628 07:52:23.935078  1594 raft_consensus.cc:2887] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:23.935757  1594 raft_consensus.cc:2270] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.937337   801 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:23.939095  1594 ts_tablet_manager.cc:1905] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.952616  1594 ts_tablet_manager.cc:1918] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:23.952967  1594 log.cc:1199] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:23.954181  1594 ts_tablet_manager.cc:1939] T d39d14572c4647c7b4880c71a8707c08 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:23.957221  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 128) took 2507 ms. Trace:
I20250628 07:52:23.957497  1594 rpcz_store.cc:276] 0628 07:52:21.449992 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.450123 (+   131us) service_pool.cc:225] Handling call
0628 07:52:23.933251 (+2483128us) ts_tablet_manager.cc:1145] Deleting tablet d39d14572c4647c7b4880c71a8707c08
0628 07:52:23.933288 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.946027 (+ 12739us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.952274 (+  6247us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.957165 (+  4891us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2479408}
I20250628 07:52:23.958565  1594 tablet_replica.cc:331] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:23.958845   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet d39d14572c4647c7b4880c71a8707c08 (table default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]) successfully deleted
I20250628 07:52:23.959477  1594 raft_consensus.cc:2241] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.960009  1594 raft_consensus.cc:2270] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.960628  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 1eac71e98ae44f41bb17816618c7d897 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:21 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:23.961694  1594 ts_tablet_manager.cc:1905] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:23.966858   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 40) took 783 ms (client timeout 1000 ms). Trace:
I20250628 07:52:23.969170  1230 raft_consensus.cc:357] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.970016  1230 raft_consensus.cc:383] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:23.970297  1230 raft_consensus.cc:738] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:23.971073  1230 consensus_queue.cc:260] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:23.974485  1594 ts_tablet_manager.cc:1918] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:23.974777  1594 log.cc:1199] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/16a8bee148664e948529d32c0cf2c793
I20250628 07:52:23.975354  1594 ts_tablet_manager.cc:1939] T 16a8bee148664e948529d32c0cf2c793 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:23.976950  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 129) took 2483 ms. Trace:
I20250628 07:52:23.977231  1594 rpcz_store.cc:276] 0628 07:52:21.493295 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.493711 (+   416us) service_pool.cc:225] Handling call
0628 07:52:23.958383 (+2464672us) ts_tablet_manager.cc:1145] Deleting tablet 16a8bee148664e948529d32c0cf2c793
0628 07:52:23.958439 (+    56us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.968184 (+  9745us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.974312 (+  6128us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.976896 (+  2584us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2459446}
I20250628 07:52:23.978193  1594 tablet_replica.cc:331] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:23.978884  1594 raft_consensus.cc:2241] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:23.979344  1594 raft_consensus.cc:2270] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:23.980895  1594 ts_tablet_manager.cc:1905] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.984382  1234 ts_tablet_manager.cc:1397] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.908s	user 0.009s	sys 0.042s
I20250628 07:52:23.995606   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d62ecf32bcc54310b9114d5655be7673" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:23.996165   821 raft_consensus.cc:3058] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:23.998132  1594 ts_tablet_manager.cc:1918] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:23.914041   758 leader_election.cc:304] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:23.998440  1594 log.cc:1199] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/83f8dcff296a4429a5f16bde59c8ebee
I20250628 07:52:23.999230  1594 ts_tablet_manager.cc:1939] T 83f8dcff296a4429a5f16bde59c8ebee P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.001984  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 133) took 2358 ms. Trace:
I20250628 07:52:24.002496  1754 raft_consensus.cc:2747] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.002277  1594 rpcz_store.cc:276] 0628 07:52:21.644025 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.644123 (+    98us) service_pool.cc:225] Handling call
0628 07:52:23.978063 (+2333940us) ts_tablet_manager.cc:1145] Deleting tablet 83f8dcff296a4429a5f16bde59c8ebee
0628 07:52:23.978101 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.991479 (+ 13378us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.997951 (+  6472us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.001921 (+  3970us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2310363}
I20250628 07:52:24.003301  1594 tablet_replica.cc:331] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:24.003258  1234 raft_consensus.cc:357] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.004007  1234 raft_consensus.cc:383] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.004103  1594 raft_consensus.cc:2241] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.004384  1234 raft_consensus.cc:738] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.004549  1594 raft_consensus.cc:2270] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.006170  1594 ts_tablet_manager.cc:1905] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:23.979928   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 16a8bee148664e948529d32c0cf2c793 (table default.Table_0adee4e1249840be952b0a99dba4c161 [id=f59d42b137cf4536a002585b728017f6]) successfully deleted
W20250628 07:52:23.981734   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 79d15e600b8b460fab54eb21193f1f40 has no consensus state
I20250628 07:52:24.008560   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 83f8dcff296a4429a5f16bde59c8ebee (table default.Table_60e2e342cb184e3da1a5b16ed3218f94 [id=9466fec47b0c4c149be47db4afeb5b31]) successfully deleted
I20250628 07:52:24.009734   540 master_service.cc:948] ReplaceTablet: received request to replace tablet ef060774191146168ee8d1e13c9cd71b from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:24.010248   546 master_service.cc:948] ReplaceTablet: received request to replace tablet 01feb8ef40dc4707bd6251fc2d87f46f from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:24.010953   544 rpcz_store.cc:269] 0628 07:52:23.183504 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.205992 (+ 22488us) service_pool.cc:225] Handling call
0628 07:52:23.966772 (+760780us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":485,"reported_tablets":20}
W20250628 07:52:24.006476   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 147168us
I20250628 07:52:24.020897  1594 ts_tablet_manager.cc:1918] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:24.021242  1594 log.cc:1199] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/1642348a16574345b31cebda23575102
I20250628 07:52:24.021898  1594 ts_tablet_manager.cc:1939] T 1642348a16574345b31cebda23575102 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.005054  1234 consensus_queue.cc:260] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.023584  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 134) took 2378 ms. Trace:
I20250628 07:52:24.023976  1594 rpcz_store.cc:276] 0628 07:52:21.645589 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.645751 (+   162us) service_pool.cc:225] Handling call
0628 07:52:24.003161 (+2357410us) ts_tablet_manager.cc:1145] Deleting tablet 1642348a16574345b31cebda23575102
0628 07:52:24.003197 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.013121 (+  9924us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.020718 (+  7597us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.023537 (+  2819us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2335060}
I20250628 07:52:24.025388  1229 tablet_bootstrap.cc:654] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.024925  1594 tablet_replica.cc:331] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:24.031843   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 1642348a16574345b31cebda23575102 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
I20250628 07:52:24.032099  1594 raft_consensus.cc:2241] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:24.033057  1594 raft_consensus.cc:2270] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:23.979635   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 56696.3us
I20250628 07:52:24.034833  1719 tablet.cc:1722] T 8c1491faa1be485792b6193d4ca22f37 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:24.035259  1594 ts_tablet_manager.cc:1905] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:24.045753   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 79d15e600b8b460fab54eb21193f1f40 has no consensus state
I20250628 07:52:24.055871  1765 tablet.cc:1722] T 8c1491faa1be485792b6193d4ca22f37 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:23.989758  1640 ts_tablet_manager.cc:1918] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:24.057428  1640 log.cc:1199] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/29acadb819ef46318e2ba5ba2d3c3c37
I20250628 07:52:24.058030  1594 ts_tablet_manager.cc:1918] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:24.056941  1230 ts_tablet_manager.cc:1428] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.123s	user 0.006s	sys 0.001s
I20250628 07:52:24.058351  1594 log.cc:1199] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/1eac71e98ae44f41bb17816618c7d897
I20250628 07:52:24.059128  1623 raft_consensus.cc:491] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.059743  1230 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:24.059528  1623 raft_consensus.cc:513] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.061297  1623 leader_election.cc:290] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:24.058147  1640 ts_tablet_manager.cc:1939] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:24.063673   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:24.064529  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 149) took 2232 ms. Trace:
W20250628 07:52:24.064862  1062 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.065447  1594 ts_tablet_manager.cc:1939] T 1eac71e98ae44f41bb17816618c7d897 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.064846  1640 rpcz_store.cc:276] 0628 07:52:21.832577 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.832706 (+   129us) service_pool.cc:225] Handling call
0628 07:52:23.787451 (+1954745us) ts_tablet_manager.cc:1145] Deleting tablet 29acadb819ef46318e2ba5ba2d3c3c37
0628 07:52:23.787491 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:23.855362 (+ 67871us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:23.989500 (+134138us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.064478 (+ 74978us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1954247}
I20250628 07:52:24.066833   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 29acadb819ef46318e2ba5ba2d3c3c37 (table default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]) successfully deleted
I20250628 07:52:24.068873   819 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:24.069964  1062 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.070297  1062 leader_election.cc:304] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.070758  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 137) took 2243 ms. Trace:
I20250628 07:52:24.071286  1623 raft_consensus.cc:2747] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.071045  1594 rpcz_store.cc:276] 0628 07:52:21.827480 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.827724 (+   244us) service_pool.cc:225] Handling call
0628 07:52:24.024803 (+2197079us) ts_tablet_manager.cc:1145] Deleting tablet 1eac71e98ae44f41bb17816618c7d897
0628 07:52:24.024837 (+    34us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.043199 (+ 18362us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.057829 (+ 14630us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.070708 (+ 12879us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":4174,"tablet-delete.queue_time_us":2196456}
I20250628 07:52:24.069729  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 148) took 2238 ms. Trace:
I20250628 07:52:24.072247   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 1eac71e98ae44f41bb17816618c7d897 (table default.Table_bcf5c3bfda174712bc8205e0bbbfe24b [id=516a02e7031940daab059a9e755eca3e]) successfully deleted
I20250628 07:52:24.072263  1640 rpcz_store.cc:276] 0628 07:52:21.831606 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:21.841329 (+  9723us) service_pool.cc:225] Handling call
0628 07:52:24.069509 (+2228180us) ts_tablet_manager.cc:1145] Deleting tablet d62ecf32bcc54310b9114d5655be7673
0628 07:52:24.069545 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.069683 (+   138us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2227170}
W20250628 07:52:24.073293   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d62ecf32bcc54310b9114d5655be7673 because the tablet was not found. No further retry: Not found: Tablet not found: d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:24.078429  1640 tablet_replica.cc:331] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:24.079653  1640 raft_consensus.cc:2241] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.080631  1640 raft_consensus.cc:2270] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.082254  1640 ts_tablet_manager.cc:1905] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.081049  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 147) took 1565 ms. Trace:
I20250628 07:52:24.080263  1230 tablet_bootstrap.cc:654] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.085845  1594 rpcz_store.cc:276] 0628 07:52:22.515869 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.517189 (+  1320us) service_pool.cc:225] Handling call
0628 07:52:24.080740 (+1563551us) ts_tablet_manager.cc:1145] Deleting tablet e4e02676d24a4345908dbbc42fc9c1ab
0628 07:52:24.080782 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.081001 (+   219us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1562750}
I20250628 07:52:24.086954  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 149) took 1560 ms. Trace:
W20250628 07:52:24.087204   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e4e02676d24a4345908dbbc42fc9c1ab because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e4e02676d24a4345908dbbc42fc9c1ab already in progress: creating tablet
I20250628 07:52:24.087244  1594 rpcz_store.cc:276] 0628 07:52:22.527039 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.537133 (+ 10094us) service_pool.cc:225] Handling call
0628 07:52:24.086638 (+1549505us) ts_tablet_manager.cc:1145] Deleting tablet fab31b073e2e49318b4118851f81b6d2
0628 07:52:24.086675 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.086901 (+   226us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1549023}
I20250628 07:52:24.088143  1594 tablet_replica.cc:331] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
W20250628 07:52:24.088311   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fab31b073e2e49318b4118851f81b6d2 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fab31b073e2e49318b4118851f81b6d2 already in progress: creating tablet
I20250628 07:52:24.088958  1594 raft_consensus.cc:2241] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:24.089802  1594 pending_rounds.cc:70] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:24.090022  1594 pending_rounds.cc:77] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493899228573696 op_type: NO_OP noop_request { }
I20250628 07:52:24.090334  1594 raft_consensus.cc:2887] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:24.090647  1594 raft_consensus.cc:2270] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.091831   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b89abfdc7dcf4f8db19baf38a00d51ef" } new_table_name: "default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a" modify_external_catalogs: true
I20250628 07:52:24.092886  1594 ts_tablet_manager.cc:1905] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.092869   539 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c6a7fbba6c24b8181422482c95fe4b7" } modify_external_catalogs: true
I20250628 07:52:24.093385   539 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8c6a7fbba6c24b8181422482c95fe4b7" } modify_external_catalogs: true
I20250628 07:52:24.097543   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7f4543a7e9114309a730d484ff513996"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.098964   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7f4543a7e9114309a730d484ff513996 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:24.112855   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet bed1ea07e3d44c08adf37ced9dbef85e consensus state has no leader
I20250628 07:52:24.118670  1234 ts_tablet_manager.cc:1428] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.124s	user 0.005s	sys 0.001s
I20250628 07:52:24.120313  1768 tablet.cc:1722] T 8c1491faa1be485792b6193d4ca22f37 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:24.125308   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_85406b18e2c04407b80adfc0e5ee9587" } modify_external_catalogs: true
I20250628 07:52:24.125777   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_85406b18e2c04407b80adfc0e5ee9587" } modify_external_catalogs: true
I20250628 07:52:24.126356  1230 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:24.126709  1230 ts_tablet_manager.cc:1397] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.068s	user 0.010s	sys 0.000s
I20250628 07:52:24.128688  1230 raft_consensus.cc:357] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.129195   821 raft_consensus.cc:2466] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:24.129371  1230 raft_consensus.cc:383] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.129613  1230 raft_consensus.cc:738] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.130255  1230 consensus_queue.cc:260] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.130717   920 leader_election.cc:304] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:24.131826  1733 raft_consensus.cc:2802] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:24.132807  1733 raft_consensus.cc:695] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:24.133630  1733 consensus_queue.cc:237] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.140030  1594 ts_tablet_manager.cc:1918] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:24.140334  1594 log.cc:1199] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/482095693aac476faf997ea8e1a4b460
I20250628 07:52:24.141177  1594 ts_tablet_manager.cc:1939] T 482095693aac476faf997ea8e1a4b460 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.141824   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b68d37d5ffd14e17820af67d5f0d7a71"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.142946  1230 ts_tablet_manager.cc:1428] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.000s
W20250628 07:52:24.143065   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b68d37d5ffd14e17820af67d5f0d7a71 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.143530  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 150) took 1606 ms. Trace:
I20250628 07:52:24.143793  1594 rpcz_store.cc:276] 0628 07:52:22.536691 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.547178 (+ 10487us) service_pool.cc:225] Handling call
0628 07:52:24.087996 (+1540818us) ts_tablet_manager.cc:1145] Deleting tablet 482095693aac476faf997ea8e1a4b460
0628 07:52:24.088049 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.129913 (+ 41864us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.139840 (+  9927us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.143485 (+  3645us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1538508}
I20250628 07:52:24.144862  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 158) took 1136 ms. Trace:
I20250628 07:52:24.144938   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 482095693aac476faf997ea8e1a4b460 (table default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:24.145131  1594 rpcz_store.cc:276] 0628 07:52:23.008581 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.008694 (+   113us) service_pool.cc:225] Handling call
0628 07:52:24.144661 (+1135967us) ts_tablet_manager.cc:1145] Deleting tablet 3223005021ae4e38b4ac65af460d0ad0
0628 07:52:24.144695 (+    34us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.144823 (+   128us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1135393}
I20250628 07:52:24.146026  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 159) took 1136 ms. Trace:
I20250628 07:52:24.146255  1594 rpcz_store.cc:276] 0628 07:52:23.009640 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.009737 (+    97us) service_pool.cc:225] Handling call
0628 07:52:24.145802 (+1136065us) ts_tablet_manager.cc:1145] Deleting tablet 2b51d6076e344c8f8e388b77b4b23928
0628 07:52:24.145837 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.145983 (+   146us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1135555}
I20250628 07:52:24.147063  1594 tablet_replica.cc:331] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
W20250628 07:52:24.147028   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 3223005021ae4e38b4ac65af460d0ad0 because the tablet was not found. No further retry: Not found: Tablet not found: 3223005021ae4e38b4ac65af460d0ad0
I20250628 07:52:24.147692  1594 raft_consensus.cc:2241] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
W20250628 07:52:24.147859   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2b51d6076e344c8f8e388b77b4b23928 because the tablet was not found. No further retry: Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:24.148434  1594 raft_consensus.cc:2270] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.148753  1230 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:24.149989  1594 ts_tablet_manager.cc:1905] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.168529  1640 ts_tablet_manager.cc:1918] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:24.168744  1244 ts_tablet_manager.cc:1918] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:24.169212  1244 log.cc:1199] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:24.170001  1244 ts_tablet_manager.cc:1939] T d39d14572c4647c7b4880c71a8707c08 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:24.170601  1640 log.cc:1199] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:24.171541  1640 ts_tablet_manager.cc:1939] T fab31b073e2e49318b4118851f81b6d2 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:24.171682  1229 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:24.172053  1229 ts_tablet_manager.cc:1397] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.244s	user 0.008s	sys 0.000s
I20250628 07:52:24.178828  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 161) took 1651 ms. Trace:
I20250628 07:52:24.170322  1230 tablet_bootstrap.cc:654] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.181803  1640 rpcz_store.cc:276] 0628 07:52:22.527894 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.528098 (+   204us) service_pool.cc:225] Handling call
0628 07:52:24.078297 (+1550199us) ts_tablet_manager.cc:1145] Deleting tablet fab31b073e2e49318b4118851f81b6d2
0628 07:52:24.078333 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.104154 (+ 25821us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.168328 (+ 64174us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.178771 (+ 10443us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":445,"tablet-delete.queue_time_us":1549473}
I20250628 07:52:24.183209   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fab31b073e2e49318b4118851f81b6d2 (table default.Table_1935e83524ad453c8175710332a3c459 [id=491d07dd82d54928b39605ba6d28af42]) successfully deleted
I20250628 07:52:24.184017  1640 tablet_replica.cc:331] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:24.185901  1594 ts_tablet_manager.cc:1918] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:24.186321  1594 log.cc:1199] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/5e1b0dbbc21b445d8115d963f6aba2a1
I20250628 07:52:24.186874  1594 ts_tablet_manager.cc:1939] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.188484  1594 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 160) took 1177 ms. Trace:
I20250628 07:52:24.188874  1594 rpcz_store.cc:276] 0628 07:52:23.010712 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.010807 (+    95us) service_pool.cc:225] Handling call
0628 07:52:24.146940 (+1136133us) ts_tablet_manager.cc:1145] Deleting tablet 5e1b0dbbc21b445d8115d963f6aba2a1
0628 07:52:24.146975 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.165268 (+ 18293us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.185723 (+ 20455us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.188429 (+  2706us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1135492}
I20250628 07:52:24.189944   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (table default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]) successfully deleted
W20250628 07:52:24.191659   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0dffb3eb778946869b914e5b05a67a69 already in progress: creating tablet
W20250628 07:52:24.192804   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 5ec8c1d9386a4922b55df5e06039858d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5ec8c1d9386a4922b55df5e06039858d already in progress: creating tablet
W20250628 07:52:24.194012   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fa96605dbabe4f0fbc26b057232960a4 because the tablet was not found. No further retry: Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:24.194685  1594 tablet_replica.cc:331] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:24.194756  1229 raft_consensus.cc:357] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.195293  1594 raft_consensus.cc:2241] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.195376  1229 raft_consensus.cc:383] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.195621  1229 raft_consensus.cc:738] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.196250  1229 consensus_queue.cc:260] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.195674  1594 raft_consensus.cc:2270] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.212889   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_88c945c579fe4aaf9c7acecfc6b88288"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.214332   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_88c945c579fe4aaf9c7acecfc6b88288 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.212990   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4504a00fdfde4d15becbe6a8a479cd0e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.217658   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_019f48b4d4734b4ba9d765410d4ebe1d" } new_table_name: "default.Table_449fb44a73414a90954c0a9ce0904745" modify_external_catalogs: true
W20250628 07:52:24.218220   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4504a00fdfde4d15becbe6a8a479cd0e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.193820  1754 raft_consensus.cc:491] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.225348  1754 raft_consensus.cc:513] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.226987  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 146) took 2125 ms. Trace:
I20250628 07:52:24.227253  1754 leader_election.cc:290] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.227309  1244 rpcz_store.cc:276] 0628 07:52:22.101292 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.101403 (+   111us) service_pool.cc:225] Handling call
0628 07:52:23.693052 (+1591649us) ts_tablet_manager.cc:1145] Deleting tablet d39d14572c4647c7b4880c71a8707c08
0628 07:52:23.716673 (+ 23621us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.010563 (+293890us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.168541 (+157978us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.226919 (+ 58378us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1568038}
I20250628 07:52:24.228454   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet d39d14572c4647c7b4880c71a8707c08 (table default.Table_b7d509011ff1418c97d34d3e92013229 [id=15d77a9fc2b64c169a7f842e6d54c909]) successfully deleted
W20250628 07:52:24.230425   757 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 16a8bee148664e948529d32c0cf2c793
I20250628 07:52:24.217203  1234 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:24.234045   758 leader_election.cc:343] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 16a8bee148664e948529d32c0cf2c793
I20250628 07:52:24.234421   758 leader_election.cc:304] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.235638  1754 raft_consensus.cc:2747] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.184803  1640 raft_consensus.cc:2241] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.242090  1640 raft_consensus.cc:2270] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:24.243094   541 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 63115b7be4a94b639c18601d438ed947 has no consensus state
I20250628 07:52:24.246237  1640 ts_tablet_manager.cc:1905] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.246644  1594 ts_tablet_manager.cc:1905] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.252445  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 147) took 2150 ms. Trace:
I20250628 07:52:24.246735  1229 ts_tablet_manager.cc:1428] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.074s	user 0.006s	sys 0.000s
I20250628 07:52:24.264688  1230 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:24.265161  1230 ts_tablet_manager.cc:1397] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.122s	user 0.015s	sys 0.000s
I20250628 07:52:24.265774  1229 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:24.267452  1230 raft_consensus.cc:357] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.268038  1230 raft_consensus.cc:383] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.268262  1230 raft_consensus.cc:738] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.268934  1230 consensus_queue.cc:260] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.261267  1234 tablet_bootstrap.cc:654] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.270258  1594 ts_tablet_manager.cc:1918] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:24.270632  1594 log.cc:1199] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/29acadb819ef46318e2ba5ba2d3c3c37
I20250628 07:52:24.272197  1594 ts_tablet_manager.cc:1939] T 29acadb819ef46318e2ba5ba2d3c3c37 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.266016  1244 rpcz_store.cc:276] 0628 07:52:22.102195 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.102290 (+    95us) service_pool.cc:225] Handling call
0628 07:52:24.252155 (+2149865us) ts_tablet_manager.cc:1145] Deleting tablet fa96605dbabe4f0fbc26b057232960a4
0628 07:52:24.252209 (+    54us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.252392 (+   183us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2126949}
W20250628 07:52:24.275925   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fa96605dbabe4f0fbc26b057232960a4 because the tablet was not found. No further retry: Not found: Tablet not found: fa96605dbabe4f0fbc26b057232960a4
I20250628 07:52:24.277921   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 29acadb819ef46318e2ba5ba2d3c3c37 (table default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]) successfully deleted
W20250628 07:52:24.278486   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9a248bf8a62b4b4eb206c0daff5e0568 already in progress: creating tablet
W20250628 07:52:24.279141   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet b51b51eb951d45a78bd2bffe47834b95 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b51b51eb951d45a78bd2bffe47834b95 already in progress: creating tablet
I20250628 07:52:24.287083   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:24.289513   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8c1491faa1be485792b6193d4ca22f37 (table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a]). Expected version 1 got 0
I20250628 07:52:24.302673   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:24.303375   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:24.307866  1229 tablet_bootstrap.cc:654] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.316644   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_845c6c9bfdfe4c98a5ff74e4547dcb2c" } new_table_name: "default.Table_7d65816a79cc4f9faa8d84ed157ed333" modify_external_catalogs: true
I20250628 07:52:24.318116  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (DEFAULT_TABLE table=default.Table_78e2ac50470645f49e4cadc30ff61a59 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.320097  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46ff641001cd4b01a4c09c0d8b79ebf3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.327391   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:24.328946   546 catalog_manager.cc:6652] ReplaceTablet: tablet 01feb8ef40dc4707bd6251fc2d87f46f deleted and replaced by tablet a447edf0960c4739930450a8c24ff6f7
I20250628 07:52:24.330051  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f with delete_type TABLET_DATA_DELETED (replaced by tablet a447edf0960c4739930450a8c24ff6f7) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:24.331466  1101 tablet_service.cc:1468] Processing CreateTablet for tablet ee03996105074308bda56d36ba3a1c5d (DEFAULT_TABLE table=default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.332482  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ee03996105074308bda56d36ba3a1c5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.338472   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]). Expected version 1 got 0
I20250628 07:52:24.339524   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8c1491faa1be485792b6193d4ca22f37 (table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a]). Expected version 1 got 0
I20250628 07:52:24.340080   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]). Expected version 1 got 0
I20250628 07:52:24.340265   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5baf5e16d9f6482ab94b15a2706e3fbd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.342134   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5baf5e16d9f6482ab94b15a2706e3fbd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.346596   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]). Expected version 1 got 0
I20250628 07:52:24.314637   965 tablet_service.cc:1468] Processing CreateTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (DEFAULT_TABLE table=default.Table_78e2ac50470645f49e4cadc30ff61a59 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.351634   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:24.351747   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46ff641001cd4b01a4c09c0d8b79ebf3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.351904   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a] alter complete (version 1)
I20250628 07:52:24.354867  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:24.330331   960 tablet_service.cc:1468] Processing CreateTablet for tablet ee03996105074308bda56d36ba3a1c5d (DEFAULT_TABLE table=default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.357695   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.357868   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ee03996105074308bda56d36ba3a1c5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.358640   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:24.360772  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:24.334723   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f with delete_type TABLET_DATA_DELETED (replaced by tablet a447edf0960c4739930450a8c24ff6f7) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.363027   964 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.367615  1623 raft_consensus.cc:491] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.368001  1623 raft_consensus.cc:513] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.368722  1230 ts_tablet_manager.cc:1428] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.103s	user 0.005s	sys 0.000s
I20250628 07:52:24.369661  1623 leader_election.cc:290] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:24.371254   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:24.372412  1230 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:24.372613  1062 leader_election.cc:343] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.377844   799 tablet_service.cc:1468] Processing CreateTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (DEFAULT_TABLE table=default.Table_78e2ac50470645f49e4cadc30ff61a59 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.378095   805 tablet_service.cc:1468] Processing CreateTablet for tablet ee03996105074308bda56d36ba3a1c5d (DEFAULT_TABLE table=default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.378544  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 2f8099bd0c9649b492edd8b657fd19e3 (DEFAULT_TABLE table=default.Table_3f52d53f779c43b38b6cd8d7332cd51f [id=8199977a96084027883e83d166d4533c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.378938   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 46ff641001cd4b01a4c09c0d8b79ebf3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.379170  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 95fa7c0a68d34d728042fb5a8a3b7307 (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.379519  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f8099bd0c9649b492edd8b657fd19e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.379552   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.380040  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 95fa7c0a68d34d728042fb5a8a3b7307. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.380383   801 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.381328   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f with delete_type TABLET_DATA_DELETED (replaced by tablet a447edf0960c4739930450a8c24ff6f7) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.381860  1108 tablet_service.cc:1468] Processing CreateTablet for tablet fef70db0996f48c69574337f8f7ecf37 (DEFAULT_TABLE table=default.Table_96f7aea7a2b445139de2bf323b687f73 [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.382671  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fef70db0996f48c69574337f8f7ecf37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.382433   800 tablet_service.cc:1468] Processing CreateTablet for tablet fef70db0996f48c69574337f8f7ecf37 (DEFAULT_TABLE table=default.Table_96f7aea7a2b445139de2bf323b687f73 [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.389750   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fef70db0996f48c69574337f8f7ecf37. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:24.390157   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 83ffa65248fc4db3aecc5f91b648b83f already in progress: creating tablet
W20250628 07:52:24.390794   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 01feb8ef40dc4707bd6251fc2d87f46f because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 01feb8ef40dc4707bd6251fc2d87f46f already in progress: creating tablet
I20250628 07:52:24.392472   964 tablet_service.cc:1468] Processing CreateTablet for tablet fef70db0996f48c69574337f8f7ecf37 (DEFAULT_TABLE table=default.Table_96f7aea7a2b445139de2bf323b687f73 [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.393420   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fef70db0996f48c69574337f8f7ecf37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.393790   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ee03996105074308bda56d36ba3a1c5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.400225   963 tablet_service.cc:1468] Processing CreateTablet for tablet 2f8099bd0c9649b492edd8b657fd19e3 (DEFAULT_TABLE table=default.Table_3f52d53f779c43b38b6cd8d7332cd51f [id=8199977a96084027883e83d166d4533c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.401115   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f8099bd0c9649b492edd8b657fd19e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.409690   961 tablet_service.cc:1468] Processing CreateTablet for tablet 95fa7c0a68d34d728042fb5a8a3b7307 (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.410702   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 95fa7c0a68d34d728042fb5a8a3b7307. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.413267  1244 tablet_replica.cc:331] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:24.413779   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:24.414113  1244 raft_consensus.cc:2241] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.414523  1244 raft_consensus.cc:2270] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:24.414901  1062 leader_election.cc:343] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.415282  1062 leader_election.cc:304] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.416563  1244 ts_tablet_manager.cc:1905] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.417218  1623 raft_consensus.cc:2747] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.421931   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:24.423126   540 catalog_manager.cc:6652] ReplaceTablet: tablet ef060774191146168ee8d1e13c9cd71b deleted and replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81
W20250628 07:52:24.423218   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 46e0aa090f544e8aa0a426c61925f345 consensus state has no leader
I20250628 07:52:24.423341   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_026074aa331e455ebaaf270237f94590"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.424391  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:24.425442   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_026074aa331e455ebaaf270237f94590 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.425588  1234 tablet_bootstrap.cc:492] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:24.425671   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because the tablet was not found. No further retry: Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:24.426021  1234 ts_tablet_manager.cc:1397] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.209s	user 0.011s	sys 0.001s
I20250628 07:52:24.427043   965 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.428118  1234 raft_consensus.cc:357] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.428745  1234 raft_consensus.cc:383] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.429009  1234 raft_consensus.cc:738] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.429719  1234 consensus_queue.cc:260] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:24.434548   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ef060774191146168ee8d1e13c9cd71b because tablet deleting was already in progress. No further retry: Already present: State transition of tablet ef060774191146168ee8d1e13c9cd71b already in progress: creating tablet
I20250628 07:52:24.438570  1229 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:24.438961  1229 ts_tablet_manager.cc:1397] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.173s	user 0.007s	sys 0.005s
I20250628 07:52:24.445643  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 08afbaee7cb74ce8a6b2958a6828eca1 (DEFAULT_TABLE table=default.Table_d2097dcb437041b98aef0c014b4e34e9 [id=89aacd4ca70d47efaf90b5662fe56b85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.446732  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08afbaee7cb74ce8a6b2958a6828eca1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.456465  1229 raft_consensus.cc:357] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.457755   965 tablet_service.cc:1468] Processing CreateTablet for tablet 08afbaee7cb74ce8a6b2958a6828eca1 (DEFAULT_TABLE table=default.Table_d2097dcb437041b98aef0c014b4e34e9 [id=89aacd4ca70d47efaf90b5662fe56b85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.458701   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08afbaee7cb74ce8a6b2958a6828eca1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.462389  1640 ts_tablet_manager.cc:1918] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:24.462725  1640 log.cc:1199] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/482095693aac476faf997ea8e1a4b460
I20250628 07:52:24.463276  1640 ts_tablet_manager.cc:1939] T 482095693aac476faf997ea8e1a4b460 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:24.473088   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1d25d1c32a764b60a6265908bd978817" } new_table_name: "default.Table_cfe73367b7384124927ff17e68820937" modify_external_catalogs: true
I20250628 07:52:24.474932   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_88ba7342f09845ad8104b1f6128cbb03"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.476220   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_88ba7342f09845ad8104b1f6128cbb03 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.483561  1229 raft_consensus.cc:383] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.483850  1229 raft_consensus.cc:738] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.486732  1754 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.487224  1754 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.491617   798 tablet_service.cc:1468] Processing CreateTablet for tablet 95fa7c0a68d34d728042fb5a8a3b7307 (DEFAULT_TABLE table=default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.492574   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 95fa7c0a68d34d728042fb5a8a3b7307. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.492349  1229 consensus_queue.cc:260] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.505306   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff] alter complete (version 1)
W20250628 07:52:24.507337   538 catalog_manager.cc:5774] Failed to send AlterTable RPC for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6], current schema version=3): Not found: failed to reset TS proxy: No leader found: Tablet 8f7dde9b060f4d839724e5f718519fb1 consensus state has no leader
I20250628 07:52:24.511088  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:24.511250   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2ae890710f7f44cf899dd19e96fa9181 consensus state has no leader
W20250628 07:52:24.513134   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 38) took 787 ms (client timeout 1000 ms). Trace:
I20250628 07:52:24.514765  1623 raft_consensus.cc:491] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.515244  1623 raft_consensus.cc:513] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.516950  1623 leader_election.cc:290] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:24.518023   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:24.518173   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:24.519958  1062 leader_election.cc:343] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.531816   966 tablet_service.cc:1515] Processing DeleteTablet for tablet d62ecf32bcc54310b9114d5655be7673 with delete_type TABLET_DATA_DELETED (replaced by tablet 9ba2a10acdf0462bb05628351cdbbbe4) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.532495  1594 tablet_replica.cc:331] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:24.533370  1229 ts_tablet_manager.cc:1428] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.094s	user 0.008s	sys 0.000s
I20250628 07:52:24.533607  1594 raft_consensus.cc:2241] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:24.534240  1594 pending_rounds.cc:70] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:24.534353  1229 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:24.534447  1594 pending_rounds.cc:77] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493902380707840 op_type: NO_OP noop_request { }
I20250628 07:52:24.534821  1594 raft_consensus.cc:2887] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:24.535102  1594 raft_consensus.cc:2270] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.538693  1594 ts_tablet_manager.cc:1905] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.540529   960 tablet_service.cc:1515] Processing DeleteTablet for tablet c84f2570b2da45d0b3ce6428723263a5 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:13 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.541119   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.542594   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 1eac71e98ae44f41bb17816618c7d897 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:21 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.545616   964 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.547480   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 482095693aac476faf997ea8e1a4b460 with delete_type TABLET_DATA_DELETED (replaced by tablet af13b56df157443c994f747e52b0f458) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:24.513412   543 rpcz_store.cc:269] 0628 07:52:23.726127 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.726261 (+   134us) service_pool.cc:225] Handling call
0628 07:52:23.807343 (+ 81082us) spinlock_profiling.cc:243] Waited 78.4 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:24.513065 (+705722us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:24.308929 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:24.309293 (+   364us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:24.309312 (+    19us) write_op.cc:435] Acquired schema lock
0628 07:52:24.309323 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:24.309524 (+   201us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:24.309552 (+    28us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:24.309571 (+    19us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:24.309685 (+   114us) tablet.cc:699] Row locks acquired
0628 07:52:24.309696 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:24.309780 (+    84us) write_op.cc:270] Start()
0628 07:52:24.309844 (+    64us) write_op.cc:276] Timestamp: P: 1751097144309707 usec, L: 0
0628 07:52:24.309857 (+    13us) op_driver.cc:348] REPLICATION: starting
0628 07:52:24.310205 (+   348us) log.cc:844] Serialized 356 byte log entry
0628 07:52:24.346831 (+ 36626us) op_driver.cc:464] REPLICATION: finished
0628 07:52:24.350191 (+  3360us) write_op.cc:301] APPLY: starting
0628 07:52:24.350230 (+    39us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:24.350305 (+    75us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:24.350369 (+    64us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:24.350534 (+   165us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:24.350662 (+   128us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:24.350709 (+    47us) write_op.cc:312] APPLY: finished
0628 07:52:24.350837 (+   128us) log.cc:844] Serialized 23 byte log entry
0628 07:52:24.350937 (+   100us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:24.350984 (+    47us) write_op.cc:454] Released schema lock
0628 07:52:24.351009 (+    25us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:24.444077 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:24.444509 (+   432us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:24.444539 (+    30us) write_op.cc:435] Acquired schema lock
0628 07:52:24.444557 (+    18us) tablet.cc:660] Decoding operations
0628 07:52:24.482256 (+ 37699us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:24.482288 (+    32us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:24.483383 (+  1095us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:24.483510 (+   127us) tablet.cc:699] Row locks acquired
0628 07:52:24.483522 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:24.483621 (+    99us) write_op.cc:270] Start()
0628 07:52:24.483690 (+    69us) write_op.cc:276] Timestamp: P: 1751097144483529 usec, L: 0
0628 07:52:24.483705 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:24.484077 (+   372us) log.cc:844] Serialized 356 byte log entry
0628 07:52:24.501598 (+ 17521us) op_driver.cc:464] REPLICATION: finished
0628 07:52:24.502007 (+   409us) write_op.cc:301] APPLY: starting
0628 07:52:24.502057 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:24.502135 (+    78us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:24.502219 (+    84us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:24.502337 (+   118us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:24.502460 (+   123us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:24.502501 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:24.502645 (+   144us) log.cc:844] Serialized 23 byte log entry
0628 07:52:24.502753 (+   108us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:24.502799 (+    46us) write_op.cc:454] Released schema lock
0628 07:52:24.502823 (+    24us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:24.502987 (+   164us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:24.503008 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":13884,"reported_tablets":14,"spinlock_wait_cycles":180349696,"child_traces":[["op",{"apply.queue_time_us":3268,"apply.run_cpu_time_us":965,"apply.run_wall_time_us":965,"num_ops":1,"prepare.queue_time_us":11343,"prepare.run_cpu_time_us":1507,"prepare.run_wall_time_us":3548,"raft.queue_time_us":63,"raft.run_cpu_time_us":9,"raft.run_wall_time_us":11,"replication_time_us":36918,"thread_start_us":71779,"threads_started":1}],["op",{"apply.queue_time_us":283,"apply.run_cpu_time_us":960,"apply.run_wall_time_us":960,"mutex_wait_us":8517,"num_ops":1,"prepare.queue_time_us":86644,"prepare.run_cpu_time_us":1684,"prepare.run_wall_time_us":40189,"replication_time_us":17818}]]}
I20250628 07:52:24.554935   805 tablet_service.cc:1468] Processing CreateTablet for tablet 2f8099bd0c9649b492edd8b657fd19e3 (DEFAULT_TABLE table=default.Table_3f52d53f779c43b38b6cd8d7332cd51f [id=8199977a96084027883e83d166d4533c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.555694  1102 tablet_service.cc:1468] Processing CreateTablet for tablet af13b56df157443c994f747e52b0f458 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.557003  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af13b56df157443c994f747e52b0f458. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.558579   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_73dcf73d62f245089f066d28320dae9c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.559960   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_73dcf73d62f245089f066d28320dae9c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.561978   966 tablet_service.cc:1468] Processing CreateTablet for tablet af13b56df157443c994f747e52b0f458 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.562929   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af13b56df157443c994f747e52b0f458. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.567162  1234 ts_tablet_manager.cc:1428] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.141s	user 0.004s	sys 0.000s
I20250628 07:52:24.569034   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4504a00fdfde4d15becbe6a8a479cd0e" } modify_external_catalogs: true
I20250628 07:52:24.569564   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4504a00fdfde4d15becbe6a8a479cd0e" } modify_external_catalogs: true
I20250628 07:52:24.570717  1234 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:24.575474   542 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:24.577404   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3" } modify_external_catalogs: true
I20250628 07:52:24.577857   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3" } modify_external_catalogs: true
I20250628 07:52:24.587463  1230 tablet_bootstrap.cc:654] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.597966  1594 ts_tablet_manager.cc:1918] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
W20250628 07:52:24.598105  1062 leader_election.cc:343] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.598331  1594 log.cc:1199] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:24.598497  1062 leader_election.cc:304] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.599423  1623 raft_consensus.cc:2747] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.600963   962 tablet_service.cc:1468] Processing CreateTablet for tablet aea9552ebd5949c8b0b5f23e132e9b6c (DEFAULT_TABLE table=default.Table_6c95fbef63ae4a34aff4518c2d8cbdad [id=4accce061ee5447187ab61870ffc1e77]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.601922   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aea9552ebd5949c8b0b5f23e132e9b6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.605346   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2f8099bd0c9649b492edd8b657fd19e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.606289   963 tablet_service.cc:1468] Processing CreateTablet for tablet 79d15e600b8b460fab54eb21193f1f40 (DEFAULT_TABLE table=default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.607177   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79d15e600b8b460fab54eb21193f1f40. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.607945  1229 tablet_bootstrap.cc:654] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:24.609530   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 39) took 767 ms (client timeout 1000 ms). Trace:
W20250628 07:52:24.609898   545 rpcz_store.cc:269] 0628 07:52:23.842316 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.842438 (+   122us) service_pool.cc:225] Handling call
0628 07:52:24.609467 (+767029us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":15443,"reported_tablets":15}
I20250628 07:52:24.614537  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 162) took 2079 ms. Trace:
I20250628 07:52:24.614842  1640 rpcz_store.cc:276] 0628 07:52:22.534807 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.534952 (+   145us) service_pool.cc:225] Handling call
0628 07:52:24.183864 (+1648912us) ts_tablet_manager.cc:1145] Deleting tablet 482095693aac476faf997ea8e1a4b460
0628 07:52:24.183909 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.354725 (+170816us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.462192 (+107467us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.614480 (+152288us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1648378}
I20250628 07:52:24.615782  1640 tablet_replica.cc:331] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:24.617552   801 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.617591   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 482095693aac476faf997ea8e1a4b460 (table default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:24.618892   801 tablet_service.cc:1468] Processing CreateTablet for tablet 08afbaee7cb74ce8a6b2958a6828eca1 (DEFAULT_TABLE table=default.Table_d2097dcb437041b98aef0c014b4e34e9 [id=89aacd4ca70d47efaf90b5662fe56b85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.619805   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08afbaee7cb74ce8a6b2958a6828eca1. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.622081  1754 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.622970  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:24.623634  1125 raft_consensus.cc:2391] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:24.632380   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:24.633239   983 raft_consensus.cc:2391] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:24.640542   544 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet f925dc5a1dbf4a6d8b35a47ca2f7ce9a
I20250628 07:52:24.640502  1733 raft_consensus.cc:491] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.640862  1733 raft_consensus.cc:513] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.641407  1640 raft_consensus.cc:2241] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.641840  1640 raft_consensus.cc:2270] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.642627  1733 leader_election.cc:290] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.643162   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_055d241428174f7891615d2422293f39" } new_table_name: "default.Table_cf53da78d4564be08dd5673e1b5a895b" modify_external_catalogs: true
I20250628 07:52:24.643455  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 79d15e600b8b460fab54eb21193f1f40 (DEFAULT_TABLE table=default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.644591  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79d15e600b8b460fab54eb21193f1f40. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.645565  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:24.646114  1125 raft_consensus.cc:2466] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:24.647146   921 leader_election.cc:304] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:24.647886  1733 raft_consensus.cc:2802] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:24.648190  1733 raft_consensus.cc:491] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:24.648481  1733 raft_consensus.cc:3058] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.655407  1733 raft_consensus.cc:513] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.655565  1106 tablet_service.cc:1468] Processing CreateTablet for tablet aea9552ebd5949c8b0b5f23e132e9b6c (DEFAULT_TABLE table=default.Table_6c95fbef63ae4a34aff4518c2d8cbdad [id=4accce061ee5447187ab61870ffc1e77]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.656658  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aea9552ebd5949c8b0b5f23e132e9b6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.658314  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:24.658811  1125 raft_consensus.cc:3058] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.659016  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 39ba8057357241b99882e883f6af93e7 (DEFAULT_TABLE table=default.Table_de48ef8f76e3475a8ec0919c675b8711 [id=9f256f66bce3433d9f694f07c12964d1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.660121  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39ba8057357241b99882e883f6af93e7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.665210  1733 leader_election.cc:290] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.643443  1640 ts_tablet_manager.cc:1905] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.665017  1125 raft_consensus.cc:2466] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:24.668643  1230 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:24.669062  1230 ts_tablet_manager.cc:1397] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.297s	user 0.007s	sys 0.003s
I20250628 07:52:24.671792  1230 raft_consensus.cc:357] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.672456  1230 raft_consensus.cc:383] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.672761  1230 raft_consensus.cc:738] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.673471  1230 consensus_queue.cc:260] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.674535  1108 tablet_service.cc:1468] Processing CreateTablet for tablet fa3ec14241e344029df3e39473181078 (DEFAULT_TABLE table=default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.678731  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 (DEFAULT_TABLE table=default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.679633  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2ae890710f7f44cf899dd19e96fa9181. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.680235  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa3ec14241e344029df3e39473181078. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.686556  1230 ts_tablet_manager.cc:1428] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.017s	user 0.002s	sys 0.003s
I20250628 07:52:24.666731   921 leader_election.cc:304] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:24.692865   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:24.697860  1779 raft_consensus.cc:2802] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:24.706427   538 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a] alter complete (version 1)
I20250628 07:52:24.709879   757 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.598974  1594 ts_tablet_manager.cc:1939] T d62ecf32bcc54310b9114d5655be7673 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:24.712777   803 tablet_service.cc:1468] Processing CreateTablet for tablet af13b56df157443c994f747e52b0f458 (DEFAULT_TABLE table=default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.713721   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af13b56df157443c994f747e52b0f458. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:24.717658   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet c4fcefaa9e954b52bbbee6e0a2f2f939 consensus state has no leader
I20250628 07:52:24.671311   960 tablet_service.cc:1468] Processing CreateTablet for tablet fa3ec14241e344029df3e39473181078 (DEFAULT_TABLE table=default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.678200   966 tablet_service.cc:1468] Processing CreateTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 (DEFAULT_TABLE table=default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.719568   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa3ec14241e344029df3e39473181078. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.720470   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2ae890710f7f44cf899dd19e96fa9181. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.687848   965 tablet_service.cc:1468] Processing CreateTablet for tablet 39ba8057357241b99882e883f6af93e7 (DEFAULT_TABLE table=default.Table_de48ef8f76e3475a8ec0919c675b8711 [id=9f256f66bce3433d9f694f07c12964d1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.723394   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39ba8057357241b99882e883f6af93e7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.697810  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:24.731783   802 tablet_service.cc:1468] Processing CreateTablet for tablet aea9552ebd5949c8b0b5f23e132e9b6c (DEFAULT_TABLE table=default.Table_6c95fbef63ae4a34aff4518c2d8cbdad [id=4accce061ee5447187ab61870ffc1e77]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.732847   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aea9552ebd5949c8b0b5f23e132e9b6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.734903  1623 raft_consensus.cc:491] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.735918  1623 raft_consensus.cc:513] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.737780  1623 leader_election.cc:290] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:24.739259   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f42b22f7096e46cf9cc09d1caecc9efb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:24.740454  1062 leader_election.cc:343] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.748558   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bc0991297d684c8fb6295c5ed29cd3c1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:24.750085   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bc0991297d684c8fb6295c5ed29cd3c1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.751119   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_60e2e342cb184e3da1a5b16ed3218f94" } new_table_name: "default.Table_ec87d950e26f4664861dc71b072bb347" modify_external_catalogs: true
I20250628 07:52:24.762177  1733 raft_consensus.cc:491] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.751288  1230 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:24.766548   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9e1de6bb51104538894960ac45e8b097"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.758287  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:24.769798   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9e1de6bb51104538894960ac45e8b097 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.778333  1733 raft_consensus.cc:513] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.773761  1229 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:24.782769  1229 ts_tablet_manager.cc:1397] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.249s	user 0.010s	sys 0.000s
W20250628 07:52:24.784255   543 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 661d92febf1142f99815f70ae9b27edc consensus state has no leader
I20250628 07:52:24.730330  1754 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.788249  1244 ts_tablet_manager.cc:1918] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:24.788753  1244 log.cc:1199] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c84f2570b2da45d0b3ce6428723263a5
I20250628 07:52:24.789008  1782 raft_consensus.cc:491] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.789770  1244 ts_tablet_manager.cc:1939] T c84f2570b2da45d0b3ce6428723263a5 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:24.789441  1782 raft_consensus.cc:513] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.791365  1782 leader_election.cc:290] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.792343   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.793416   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.793995  1594 tablet_replica.cc:331] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:24.795372   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet d62ecf32bcc54310b9114d5655be7673 (table default.Table_8dd91f20a94d4b7fa01cb1a8b96a48e0 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
W20250628 07:52:24.796211   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet c84f2570b2da45d0b3ce6428723263a5 because the tablet was not found. No further retry: Not found: Tablet not found: c84f2570b2da45d0b3ce6428723263a5
W20250628 07:52:24.796958   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 because the tablet was not found. No further retry: Not found: Tablet not found: 5e1b0dbbc21b445d8115d963f6aba2a1
W20250628 07:52:24.797724   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 1eac71e98ae44f41bb17816618c7d897 because the tablet was not found. No further retry: Not found: Tablet not found: 1eac71e98ae44f41bb17816618c7d897
W20250628 07:52:24.798482   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because the tablet was not found. No further retry: Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:24.798744  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:24.799279   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 482095693aac476faf997ea8e1a4b460 because the tablet was not found. No further retry: Not found: Tablet not found: 482095693aac476faf997ea8e1a4b460
I20250628 07:52:24.799351  1128 raft_consensus.cc:2466] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:24.801856   962 tablet_service.cc:1468] Processing CreateTablet for tablet d87dd96d00964e12acd6e380a62c94d8 (DEFAULT_TABLE table=default.Table_6ce1e375ffee4e2db52c58c171ba4d08 [id=a58302bfde464e91b75b443b8a701d24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.802394   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f42b22f7096e46cf9cc09d1caecc9efb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:24.802770   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d87dd96d00964e12acd6e380a62c94d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.806627  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:24.741648   799 tablet_service.cc:1515] Processing DeleteTablet for tablet 29acadb819ef46318e2ba5ba2d3c3c37 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:16 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.807219  1125 raft_consensus.cc:2466] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:24.808585   921 leader_election.cc:304] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:24.809324  1782 raft_consensus.cc:2802] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:24.809648  1782 raft_consensus.cc:491] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:24.809964  1782 raft_consensus.cc:3058] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.815927  1230 tablet_bootstrap.cc:654] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.809460   921 leader_election.cc:304] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:24.816476  1782 raft_consensus.cc:513] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.819310  1782 leader_election.cc:290] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:24.820484  1062 leader_election.cc:343] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.820868  1062 leader_election.cc:304] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.808346   804 tablet_service.cc:1468] Processing CreateTablet for tablet 79d15e600b8b460fab54eb21193f1f40 (DEFAULT_TABLE table=default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:24.779021   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 69260.6us
I20250628 07:52:24.815524   797 tablet_service.cc:1468] Processing CreateTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 (DEFAULT_TABLE table=default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.816079   802 tablet_service.cc:1468] Processing CreateTablet for tablet 39ba8057357241b99882e883f6af93e7 (DEFAULT_TABLE table=default.Table_de48ef8f76e3475a8ec0919c675b8711 [id=9f256f66bce3433d9f694f07c12964d1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.816247   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.821806  1623 raft_consensus.cc:2747] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.822255   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39ba8057357241b99882e883f6af93e7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.820461  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:24.822979  1125 raft_consensus.cc:3058] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.823050   801 tablet_service.cc:1468] Processing CreateTablet for tablet fa3ec14241e344029df3e39473181078 (DEFAULT_TABLE table=default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.823890   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa3ec14241e344029df3e39473181078. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.825536   795 tablet_service.cc:1468] Processing CreateTablet for tablet d87dd96d00964e12acd6e380a62c94d8 (DEFAULT_TABLE table=default.Table_6ce1e375ffee4e2db52c58c171ba4d08 [id=a58302bfde464e91b75b443b8a701d24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.826529   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d87dd96d00964e12acd6e380a62c94d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.828915  1623 raft_consensus.cc:491] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.829335  1623 raft_consensus.cc:513] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:24.830996   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9579dd44e5e241aea2671be28e16ef7c already in progress: creating tablet
I20250628 07:52:24.831111  1623 leader_election.cc:290] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:24.832806   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f29c8ffb7df410295278eee9c578b03" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:24.834668  1062 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.791674  1779 raft_consensus.cc:695] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:24.835667  1779 consensus_queue.cc:237] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.821379  1781 raft_consensus.cc:2802] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:24.837229  1781 raft_consensus.cc:491] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:24.837572  1781 raft_consensus.cc:3058] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.838102   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2ae890710f7f44cf899dd19e96fa9181. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.782898   800 tablet_service.cc:1515] Processing DeleteTablet for tablet d39d14572c4647c7b4880c71a8707c08 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:14 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.849663   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4f01715add2e42e2847f64142c132a2f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.849848  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 547d0e51544c49fcbcfdcd05f2390025 (DEFAULT_TABLE table=default.Table_f5b1b57d6e754cccb6eb0a1405289fde [id=53bd74ded2bf4f5fa0edb44f63d4a340]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.850256   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7f4543a7e9114309a730d484ff513996" } modify_external_catalogs: true
I20250628 07:52:24.850749   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7f4543a7e9114309a730d484ff513996" } modify_external_catalogs: true
I20250628 07:52:24.850945  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 547d0e51544c49fcbcfdcd05f2390025. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:24.851084   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4f01715add2e42e2847f64142c132a2f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.852975   966 tablet_service.cc:1468] Processing CreateTablet for tablet 547d0e51544c49fcbcfdcd05f2390025 (DEFAULT_TABLE table=default.Table_f5b1b57d6e754cccb6eb0a1405289fde [id=53bd74ded2bf4f5fa0edb44f63d4a340]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.853919   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 547d0e51544c49fcbcfdcd05f2390025. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.854765   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79d15e600b8b460fab54eb21193f1f40. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.860193  1234 tablet_bootstrap.cc:654] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:24.863072  1108 tablet_service.cc:1468] Processing CreateTablet for tablet d87dd96d00964e12acd6e380a62c94d8 (DEFAULT_TABLE table=default.Table_6ce1e375ffee4e2db52c58c171ba4d08 [id=a58302bfde464e91b75b443b8a701d24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.867965   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:24.869444   960 tablet_service.cc:1468] Processing CreateTablet for tablet 63115b7be4a94b639c18601d438ed947 (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.870481   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63115b7be4a94b639c18601d438ed947. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.868204  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 63115b7be4a94b639c18601d438ed947 (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.875262  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63115b7be4a94b639c18601d438ed947. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.877708   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1d2c0b7e3d9e448ebb2fe3530da7bbba" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:24.880198   819 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:24.881716  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d87dd96d00964e12acd6e380a62c94d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.883072  1754 raft_consensus.cc:2747] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.886767   543 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 7c32b053d589473b91eec063fb3ac32a
I20250628 07:52:24.829361  1733 leader_election.cc:290] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:24.790472  1229 raft_consensus.cc:357] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.833837   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f29c8ffb7df410295278eee9c578b03" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:24.891798  1229 raft_consensus.cc:383] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.892096  1229 raft_consensus.cc:738] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.892652  1229 consensus_queue.cc:260] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:24.894492   920 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.894949  1229 ts_tablet_manager.cc:1428] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.112s	user 0.004s	sys 0.000s
I20250628 07:52:24.895788  1229 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:24.895659   920 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:24.897946   920 leader_election.cc:343] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.884233  1640 ts_tablet_manager.cc:1918] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:24.905588  1640 log.cc:1199] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/e4e02676d24a4345908dbbc42fc9c1ab
I20250628 07:52:24.906308  1640 ts_tablet_manager.cc:1939] T e4e02676d24a4345908dbbc42fc9c1ab P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:24.906809   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_55f07e6cb519419fa92ea6aeab0cad74" } modify_external_catalogs: true
I20250628 07:52:24.907320   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_55f07e6cb519419fa92ea6aeab0cad74" } modify_external_catalogs: true
I20250628 07:52:24.909370  1125 raft_consensus.cc:2466] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:24.911156   921 leader_election.cc:304] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:24.911947  1782 raft_consensus.cc:2802] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:24.916092  1782 raft_consensus.cc:695] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:24.916878  1782 consensus_queue.cc:237] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.883908   819 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:24.885133   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22b47b39f8eb45a0b5d2c5348bfbc55c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:24.928242   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 2d77be1f37d2400f842c01019e4eb474
I20250628 07:52:24.928368   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f2b450a384bc4c03abafa265e98da078"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:24.930634   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 2d77be1f37d2400f842c01019e4eb474 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.931181  1229 tablet_bootstrap.cc:654] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:24.931699   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f2b450a384bc4c03abafa265e98da078 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:24.932796  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 2d77be1f37d2400f842c01019e4eb474 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:24.905607  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 148) took 2802 ms. Trace:
I20250628 07:52:24.937427  1594 raft_consensus.cc:2241] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:24.937233  1244 rpcz_store.cc:276] 0628 07:52:22.102944 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.103051 (+   107us) service_pool.cc:225] Handling call
0628 07:52:24.380769 (+2277718us) ts_tablet_manager.cc:1145] Deleting tablet c84f2570b2da45d0b3ce6428723263a5
0628 07:52:24.389167 (+  8398us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.633531 (+244364us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.787961 (+154430us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.905541 (+117580us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2151276}
I20250628 07:52:24.939121  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 159) took 2408 ms. Trace:
I20250628 07:52:24.939386  1594 pending_rounds.cc:70] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:24.939603  1594 pending_rounds.cc:77] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493905268596736 op_type: NO_OP noop_request { }
I20250628 07:52:24.939927  1594 raft_consensus.cc:2887] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:24.940212  1594 raft_consensus.cc:2270] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:24.943212  1787 consensus_queue.cc:1162] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "82f43cdfab544b17a6a71033983e65d4" status { last_received { term: 1 index: 1 } last_committed_idx: 0 last_received_current_leader { term: 1 index: 1 } }
I20250628 07:52:24.941895  1781 raft_consensus.cc:513] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:24.892638  1062 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.944950  1062 leader_election.cc:304] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.945477  1781 leader_election.cc:290] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:24.945904  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 53308.3us
I20250628 07:52:24.946416  1623 raft_consensus.cc:2747] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.947328  1594 ts_tablet_manager.cc:1905] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:24.948693  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:24.949267  1125 raft_consensus.cc:3058] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:24.949343   800 tablet_service.cc:1468] Processing CreateTablet for tablet 63115b7be4a94b639c18601d438ed947 (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.950474   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 63115b7be4a94b639c18601d438ed947. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.886746  1230 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:24.953590  1230 ts_tablet_manager.cc:1397] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.202s	user 0.005s	sys 0.005s
I20250628 07:52:24.954545   799 tablet_service.cc:1468] Processing CreateTablet for tablet 547d0e51544c49fcbcfdcd05f2390025 (DEFAULT_TABLE table=default.Table_f5b1b57d6e754cccb6eb0a1405289fde [id=53bd74ded2bf4f5fa0edb44f63d4a340]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:24.955523   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 547d0e51544c49fcbcfdcd05f2390025. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:24.955984  1230 raft_consensus.cc:357] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.961319   802 tablet_service.cc:1515] Processing DeleteTablet for tablet 2d77be1f37d2400f842c01019e4eb474 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:24.957690  1623 raft_consensus.cc:491] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:24.964890  1623 raft_consensus.cc:513] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:24.965961   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet c84f2570b2da45d0b3ce6428723263a5 (table default.Table_9be11b679e27417a8706dbdf185a798c [id=c69e3c832f4d419b8822ba7e8caa5281]) successfully deleted
I20250628 07:52:24.967280  1623 leader_election.cc:290] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:24.969681   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:24.970952  1062 leader_election.cc:343] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.972187   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:24.973701  1062 leader_election.cc:343] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.974103  1062 leader_election.cc:304] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:24.975310  1623 raft_consensus.cc:2747] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:24.976351  1230 raft_consensus.cc:383] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:24.977977  1230 raft_consensus.cc:738] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:24.981324  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 160) took 2468 ms. Trace:
I20250628 07:52:24.983238  1640 rpcz_store.cc:276] 0628 07:52:22.512876 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.540183 (+ 27307us) service_pool.cc:225] Handling call
0628 07:52:24.615645 (+2075462us) ts_tablet_manager.cc:1145] Deleting tablet e4e02676d24a4345908dbbc42fc9c1ab
0628 07:52:24.615685 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.772996 (+157311us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.882003 (+109007us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:24.981248 (+ 99245us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2074945}
W20250628 07:52:24.984009   920 leader_election.cc:343] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:24.985352  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 170) took 1834 ms. Trace:
I20250628 07:52:24.985749  1640 rpcz_store.cc:276] 0628 07:52:23.151016 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.157208 (+  6192us) service_pool.cc:225] Handling call
0628 07:52:24.985030 (+1827822us) ts_tablet_manager.cc:1145] Deleting tablet 83f8dcff296a4429a5f16bde59c8ebee
0628 07:52:24.985082 (+    52us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.985290 (+   208us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1827143}
I20250628 07:52:24.986989  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 171) took 1826 ms. Trace:
I20250628 07:52:24.987418  1640 rpcz_store.cc:276] 0628 07:52:23.160795 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.160938 (+   143us) service_pool.cc:225] Handling call
0628 07:52:24.986789 (+1825851us) ts_tablet_manager.cc:1145] Deleting tablet 5e1b0dbbc21b445d8115d963f6aba2a1
0628 07:52:24.986822 (+    33us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.986942 (+   120us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1825229}
I20250628 07:52:24.989130  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 172) took 1788 ms. Trace:
I20250628 07:52:24.989570  1640 rpcz_store.cc:276] 0628 07:52:23.200458 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.200583 (+   125us) service_pool.cc:225] Handling call
0628 07:52:24.988875 (+1788292us) ts_tablet_manager.cc:1145] Deleting tablet 1642348a16574345b31cebda23575102
0628 07:52:24.988923 (+    48us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.989053 (+   130us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1787719}
I20250628 07:52:24.991231  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 173) took 1789 ms. Trace:
I20250628 07:52:24.991547  1640 rpcz_store.cc:276] 0628 07:52:23.202145 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.202280 (+   135us) service_pool.cc:225] Handling call
0628 07:52:24.990996 (+1788716us) ts_tablet_manager.cc:1145] Deleting tablet 16a8bee148664e948529d32c0cf2c793
0628 07:52:24.991046 (+    50us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.991191 (+   145us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1788118}
I20250628 07:52:24.992962  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 174) took 1645 ms. Trace:
I20250628 07:52:24.993304  1640 rpcz_store.cc:276] 0628 07:52:23.347555 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.356543 (+  8988us) service_pool.cc:225] Handling call
0628 07:52:24.992680 (+1636137us) ts_tablet_manager.cc:1145] Deleting tablet 0dffb3eb778946869b914e5b05a67a69
0628 07:52:24.992713 (+    33us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.992920 (+   207us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1615002}
I20250628 07:52:24.994465  1640 tablet_replica.cc:331] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:24.995461  1640 raft_consensus.cc:2241] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:24.999617  1229 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:24.999708  1594 ts_tablet_manager.cc:1918] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:25.000150  1594 log.cc:1199] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:25.000135  1229 ts_tablet_manager.cc:1397] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.105s	user 0.011s	sys 0.000s
I20250628 07:52:25.001122  1594 ts_tablet_manager.cc:1939] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:25.003307  1594 tablet_replica.cc:331] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:25.003499   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:25.003901  1594 raft_consensus.cc:2241] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.004333  1594 raft_consensus.cc:2270] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:24.996436  1230 consensus_queue.cc:260] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.005460  1623 raft_consensus.cc:491] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.005883  1623 raft_consensus.cc:513] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.007081  1594 ts_tablet_manager.cc:1905] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.007577  1623 leader_election.cc:290] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:25.005429  1640 raft_consensus.cc:2270] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.010025  1640 ts_tablet_manager.cc:1905] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.010008   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9a248bf8a62b4b4eb206c0daff5e0568" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:25.011354  1062 leader_election.cc:343] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.012115   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9a248bf8a62b4b4eb206c0daff5e0568" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:25.013995  1062 leader_election.cc:343] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.014324  1062 leader_election.cc:304] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.015239  1623 raft_consensus.cc:2747] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.024190   538 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_116bc2985c3e44a1962cca8c38c0d338" } new_table_name: "default.Table_92144695519f4cc9a21fedc487712d03" modify_external_catalogs: true
I20250628 07:52:25.029348  1230 ts_tablet_manager.cc:1428] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.075s	user 0.006s	sys 0.000s
I20250628 07:52:25.023743   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e26652b4c0454211b542434dc5992cd9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.031240   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_88ba7342f09845ad8104b1f6128cbb03" } new_table_name: "default.Table_7e69244decba48b8b9a9f406ccf40827" modify_external_catalogs: true
W20250628 07:52:25.032032   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e26652b4c0454211b542434dc5992cd9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.034584  1229 raft_consensus.cc:357] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.035225  1229 raft_consensus.cc:383] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.035473  1229 raft_consensus.cc:738] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.036023  1229 consensus_queue.cc:260] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.035038   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:24.939428  1244 rpcz_store.cc:276] 0628 07:52:22.530807 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.530925 (+   118us) service_pool.cc:225] Handling call
0628 07:52:24.938773 (+2407848us) ts_tablet_manager.cc:1145] Deleting tablet fab31b073e2e49318b4118851f81b6d2
0628 07:52:24.938835 (+    62us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:24.939073 (+   238us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2383289}
I20250628 07:52:25.040419   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet e4e02676d24a4345908dbbc42fc9c1ab (table default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]) successfully deleted
W20250628 07:52:25.040998   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 83f8dcff296a4429a5f16bde59c8ebee because the tablet was not found. No further retry: Not found: Tablet not found: 83f8dcff296a4429a5f16bde59c8ebee
W20250628 07:52:25.041545   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5e1b0dbbc21b445d8115d963f6aba2a1 because the tablet was not found. No further retry: Not found: Tablet not found: 5e1b0dbbc21b445d8115d963f6aba2a1
W20250628 07:52:25.041916   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fab31b073e2e49318b4118851f81b6d2 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fab31b073e2e49318b4118851f81b6d2 already in progress: creating tablet
W20250628 07:52:25.042070   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1642348a16574345b31cebda23575102 because the tablet was not found. No further retry: Not found: Tablet not found: 1642348a16574345b31cebda23575102
W20250628 07:52:25.042750   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 16a8bee148664e948529d32c0cf2c793 because the tablet was not found. No further retry: Not found: Tablet not found: 16a8bee148664e948529d32c0cf2c793
W20250628 07:52:25.043365   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0dffb3eb778946869b914e5b05a67a69 already in progress: creating tablet
W20250628 07:52:25.046926   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 62986.5us
W20250628 07:52:25.047950   920 leader_election.cc:343] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.049984  1782 raft_consensus.cc:491] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.050393  1782 raft_consensus.cc:513] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.050913  1229 ts_tablet_manager.cc:1428] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.050s	user 0.005s	sys 0.000s
I20250628 07:52:25.051743  1229 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:25.052168  1782 leader_election.cc:290] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:25.052918  1594 ts_tablet_manager.cc:1918] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:25.053247  1594 log.cc:1199] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/2d77be1f37d2400f842c01019e4eb474
I20250628 07:52:25.053936  1594 ts_tablet_manager.cc:1939] T 2d77be1f37d2400f842c01019e4eb474 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:25.046487  1125 raft_consensus.cc:2466] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:25.052038  1640 ts_tablet_manager.cc:1918] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.057829   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]) successfully deleted
I20250628 07:52:25.058048  1640 log.cc:1199] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:25.056070  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:25.058727  1640 ts_tablet_manager.cc:1939] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:25.058641  1128 raft_consensus.cc:2466] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:25.059778   921 leader_election.cc:304] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:25.060796   921 leader_election.cc:304] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:25.061507  1779 raft_consensus.cc:2802] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:25.061839  1779 raft_consensus.cc:491] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:25.062160  1779 raft_consensus.cc:3058] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:25.065462   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba because the tablet was not found. No further retry: Not found: Tablet not found: 1d2c0b7e3d9e448ebb2fe3530da7bbba
W20250628 07:52:25.066792   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet c4fcefaa9e954b52bbbee6e0a2f2f939 consensus state has no leader
I20250628 07:52:25.066939  1782 raft_consensus.cc:2802] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:25.067713  1782 raft_consensus.cc:695] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
W20250628 07:52:25.068924   538 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet f845f32699044867af6f0f2b655cad8e has no consensus state
I20250628 07:52:25.068522  1782 consensus_queue.cc:237] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.074919  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 176) took 1691 ms. Trace:
I20250628 07:52:25.075290  1640 rpcz_store.cc:276] 0628 07:52:23.383610 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.393724 (+ 10114us) service_pool.cc:225] Handling call
0628 07:52:24.994330 (+1600606us) ts_tablet_manager.cc:1145] Deleting tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba
0628 07:52:24.994367 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.025734 (+ 31367us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.051809 (+ 26075us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.074861 (+ 23052us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1600072}
I20250628 07:52:25.076565  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 178) took 1613 ms. Trace:
I20250628 07:52:25.076529   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:25.077025  1640 rpcz_store.cc:276] 0628 07:52:23.463485 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.463622 (+   137us) service_pool.cc:225] Handling call
0628 07:52:25.076257 (+1612635us) ts_tablet_manager.cc:1145] Deleting tablet 5ec8c1d9386a4922b55df5e06039858d
0628 07:52:25.076299 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.076519 (+   220us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1612109}
I20250628 07:52:25.078899  1640 tablet_replica.cc:331] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:25.079655  1640 raft_consensus.cc:2241] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.080503  1640 raft_consensus.cc:2270] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.078356  1230 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:25.083009   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5ec8c1d9386a4922b55df5e06039858d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5ec8c1d9386a4922b55df5e06039858d already in progress: creating tablet
I20250628 07:52:25.083452  1640 ts_tablet_manager.cc:1905] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.088517  1229 tablet_bootstrap.cc:654] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.088768   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_49dd1c44ec2d43b9bdfe6c8169f90406"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.089473   539 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4f01715add2e42e2847f64142c132a2f" } modify_external_catalogs: true
I20250628 07:52:25.090018   539 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4f01715add2e42e2847f64142c132a2f" } modify_external_catalogs: true
W20250628 07:52:25.090206   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_49dd1c44ec2d43b9bdfe6c8169f90406 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:25.092541   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8e3e0e52b4924e67904d3acfa4dd5752 has no consensus state
I20250628 07:52:25.096064  1779 raft_consensus.cc:513] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.098387  1779 leader_election.cc:290] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:25.102226  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:25.104203  1125 raft_consensus.cc:3058] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:25.106472  1230 tablet_bootstrap.cc:654] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.108629   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a374b2114f7447f392f342b445cd482d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:25.110594   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a374b2114f7447f392f342b445cd482d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.123266  1230 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.123703  1230 ts_tablet_manager.cc:1397] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.046s	user 0.014s	sys 0.001s
I20250628 07:52:25.137317   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c8d2155d54dc47c0a639ace22ba757eb" } modify_external_catalogs: true
I20250628 07:52:25.138029   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c8d2155d54dc47c0a639ace22ba757eb" } modify_external_catalogs: true
I20250628 07:52:25.141835  1640 ts_tablet_manager.cc:1918] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.142254  1640 log.cc:1199] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:25.142985  1640 ts_tablet_manager.cc:1939] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:25.145357  1230 raft_consensus.cc:357] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.146282  1230 raft_consensus.cc:383] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.146571  1230 raft_consensus.cc:738] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.147595  1230 consensus_queue.cc:260] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.149516   539 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet b5cde27ee2f34e61bdfdc8674941171d
I20250628 07:52:25.149648  1230 ts_tablet_manager.cc:1428] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.025s	user 0.005s	sys 0.000s
I20250628 07:52:25.150767  1230 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.145038  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 180) took 1251 ms. Trace:
I20250628 07:52:25.154928  1640 rpcz_store.cc:276] 0628 07:52:23.893507 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.900342 (+  6835us) service_pool.cc:225] Handling call
0628 07:52:25.078753 (+1178411us) ts_tablet_manager.cc:1145] Deleting tablet 9a248bf8a62b4b4eb206c0daff5e0568
0628 07:52:25.078792 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.118521 (+ 39729us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.141594 (+ 23073us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.144972 (+  3378us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1176975}
I20250628 07:52:25.155974   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 9a248bf8a62b4b4eb206c0daff5e0568 (table default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]) successfully deleted
I20250628 07:52:25.156906  1230 tablet_bootstrap.cc:654] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.158166  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 181) took 1237 ms. Trace:
I20250628 07:52:25.163962  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:25.164007  1623 raft_consensus.cc:491] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.170631  1623 raft_consensus.cc:513] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.173166  1125 raft_consensus.cc:2466] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:25.174495  1229 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:25.174892  1229 ts_tablet_manager.cc:1397] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.123s	user 0.004s	sys 0.006s
I20250628 07:52:25.178436   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:25.179535  1782 raft_consensus.cc:491] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.179958  1782 raft_consensus.cc:513] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.182216  1782 leader_election.cc:290] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:25.184836   921 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:25.201431  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 158) took 2685 ms. Trace:
I20250628 07:52:25.201822  1244 rpcz_store.cc:276] 0628 07:52:22.516206 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.548050 (+ 31844us) service_pool.cc:225] Handling call
0628 07:52:25.183274 (+2635224us) ts_tablet_manager.cc:1145] Deleting tablet e4e02676d24a4345908dbbc42fc9c1ab
0628 07:52:25.187311 (+  4037us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.201376 (+ 14065us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2623587}
W20250628 07:52:25.203768   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet e4e02676d24a4345908dbbc42fc9c1ab because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e4e02676d24a4345908dbbc42fc9c1ab already in progress: creating tablet
I20250628 07:52:25.206693  1244 tablet_replica.cc:331] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:25.176995  1229 raft_consensus.cc:357] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:25.179821  1062 leader_election.cc:343] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.208747  1229 raft_consensus.cc:383] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.180189  1623 leader_election.cc:290] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:25.209335  1229 raft_consensus.cc:738] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.210426  1229 consensus_queue.cc:260] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.210610   921 leader_election.cc:304] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:25.211534  1779 raft_consensus.cc:2802] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:25.211935  1779 raft_consensus.cc:695] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:25.213035  1229 ts_tablet_manager.cc:1428] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.038s	user 0.007s	sys 0.000s
I20250628 07:52:25.212634  1779 consensus_queue.cc:237] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.213263  1244 raft_consensus.cc:2241] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.214727   816 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fab31b073e2e49318b4118851f81b6d2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:25.218645  1229 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:25.221277   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:25.226934   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:25.210013   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:25.206182   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e15d063a4bca40d480cba08733ec1ec3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:25.229980   920 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.230376   920 leader_election.cc:304] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:25.231313  1779 raft_consensus.cc:2747] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:25.234701   920 leader_election.cc:343] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:25.236675   920 leader_election.cc:343] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.242651  1230 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.243160  1230 ts_tablet_manager.cc:1397] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.093s	user 0.017s	sys 0.025s
I20250628 07:52:25.246140  1230 raft_consensus.cc:357] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.246752  1230 raft_consensus.cc:383] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.247072  1230 raft_consensus.cc:738] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.247718  1230 consensus_queue.cc:260] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.258105  1229 tablet_bootstrap.cc:654] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:25.261387   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 47656.4us
I20250628 07:52:25.261941  1244 raft_consensus.cc:2270] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.263732  1244 ts_tablet_manager.cc:1905] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:25.265898   920 leader_election.cc:343] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.272758   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2af9142e039c4b78a79c8f8252e66c86" } new_table_name: "default.Table_62804f58107647eb82169b3550598170" modify_external_catalogs: true
I20250628 07:52:25.273337  1230 ts_tablet_manager.cc:1428] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.030s	user 0.006s	sys 0.000s
I20250628 07:52:25.264716  1234 tablet_bootstrap.cc:492] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:25.289456  1234 ts_tablet_manager.cc:1397] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.719s	user 0.019s	sys 0.002s
I20250628 07:52:25.290536   539 master_service.cc:948] ReplaceTablet: received request to replace tablet 482095693aac476faf997ea8e1a4b460 from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:25.293916   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 7eca25db66fa42a798fc002d43f7e3c3 consensus state has no leader
I20250628 07:52:25.300381  1230 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.300493   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6da92141ad354b5491ebafee908fe6d7" } new_table_name: "default.Table_14e4b6f1c4f344a280e1901e7f772fbd" modify_external_catalogs: true
I20250628 07:52:25.303089   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_18695aa15f7843b9a5388e863015aae0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:25.304719   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_18695aa15f7843b9a5388e863015aae0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.309875  1234 raft_consensus.cc:357] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.313253  1234 raft_consensus.cc:383] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.314782  1234 raft_consensus.cc:738] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:25.317955  1062 leader_election.cc:343] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.318353  1062 leader_election.cc:304] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.319243  1623 raft_consensus.cc:2747] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.325532  1230 tablet_bootstrap.cc:654] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.329469  1234 consensus_queue.cc:260] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.344015  1640 rpcz_store.cc:276] 0628 07:52:23.920632 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.920786 (+   154us) service_pool.cc:225] Handling call
0628 07:52:25.157755 (+1236969us) ts_tablet_manager.cc:1145] Deleting tablet b51b51eb951d45a78bd2bffe47834b95
0628 07:52:25.157806 (+    51us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.158083 (+   277us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1236402}
I20250628 07:52:25.345319  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 182) took 1385 ms. Trace:
I20250628 07:52:25.345746  1640 rpcz_store.cc:276] 0628 07:52:23.960365 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.960490 (+   125us) service_pool.cc:225] Handling call
0628 07:52:25.345042 (+1384552us) ts_tablet_manager.cc:1145] Deleting tablet 1eac71e98ae44f41bb17816618c7d897
0628 07:52:25.345115 (+    73us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.345254 (+   139us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1383989}
I20250628 07:52:25.346684  1640 tablet_replica.cc:331] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:25.347107   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c81c2135bc5a4417b706d1c042207cc0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.347442  1640 raft_consensus.cc:2241] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.349740   543 master_service.cc:948] ReplaceTablet: received request to replace tablet cb62cf19aaf04444845358cc8ad611fb from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:25.350659  1640 raft_consensus.cc:2270] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.356096  1234 ts_tablet_manager.cc:1428] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.066s	user 0.003s	sys 0.002s
I20250628 07:52:25.359009   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_caf24e76219f4823a328d7048ce15d24"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.361066   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_026074aa331e455ebaaf270237f94590" } modify_external_catalogs: true
I20250628 07:52:25.361411  1230 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.361562   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_026074aa331e455ebaaf270237f94590" } modify_external_catalogs: true
I20250628 07:52:25.361990  1230 ts_tablet_manager.cc:1397] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.062s	user 0.013s	sys 0.002s
W20250628 07:52:25.346109   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b51b51eb951d45a78bd2bffe47834b95 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b51b51eb951d45a78bd2bffe47834b95 already in progress: creating tablet
W20250628 07:52:25.363600   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1eac71e98ae44f41bb17816618c7d897 because the tablet was not found. No further retry: Not found: Tablet not found: 1eac71e98ae44f41bb17816618c7d897
W20250628 07:52:25.365763   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_caf24e76219f4823a328d7048ce15d24 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:25.366144   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c81c2135bc5a4417b706d1c042207cc0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.369688  1230 raft_consensus.cc:357] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.371999  1230 raft_consensus.cc:383] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.372547  1230 raft_consensus.cc:738] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.373492  1230 consensus_queue.cc:260] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.380311  1234 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:25.389592  1230 ts_tablet_manager.cc:1428] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.027s	user 0.005s	sys 0.000s
I20250628 07:52:25.390008  1640 ts_tablet_manager.cc:1905] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.396109   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9775925de3b34654b11d221bc66cb7fd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:25.397548   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9775925de3b34654b11d221bc66cb7fd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.402673  1230 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.408273  1230 tablet_bootstrap.cc:654] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.401016  1244 ts_tablet_manager.cc:1918] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.408676  1244 log.cc:1199] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/482095693aac476faf997ea8e1a4b460
I20250628 07:52:25.409432  1244 ts_tablet_manager.cc:1939] T 482095693aac476faf997ea8e1a4b460 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:25.426815  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 160) took 2889 ms. Trace:
I20250628 07:52:25.427146   546 catalog_manager.cc:5582] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:25.428093  1230 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.428454  1230 ts_tablet_manager.cc:1397] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.026s	user 0.007s	sys 0.003s
W20250628 07:52:25.429981   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet e15d063a4bca40d480cba08733ec1ec3 consensus state has no leader
I20250628 07:52:25.431185  1230 raft_consensus.cc:357] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.431809  1230 raft_consensus.cc:383] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.435047  1229 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:25.435396  1229 ts_tablet_manager.cc:1397] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.218s	user 0.009s	sys 0.000s
I20250628 07:52:25.436730  1230 raft_consensus.cc:738] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.437950  1230 consensus_queue.cc:260] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.438643  1623 raft_consensus.cc:491] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.439003  1623 raft_consensus.cc:513] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:25.418560   757 consensus_peers.cc:489] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2d77be1f37d2400f842c01019e4eb474. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:25.417251  1234 tablet_bootstrap.cc:654] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.442394  1623 leader_election.cc:290] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:25.443704   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cb62cf19aaf04444845358cc8ad611fb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:25.443783   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cb62cf19aaf04444845358cc8ad611fb" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:25.445334  1062 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.445330  1229 raft_consensus.cc:357] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:25.445891  1062 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.445966  1229 raft_consensus.cc:383] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.446182  1229 raft_consensus.cc:738] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.446188  1062 leader_election.cc:304] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.449043  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 (DEFAULT_TABLE table=default.Table_73dcf73d62f245089f066d28320dae9c [id=ab01e12b48e047828205036a96ce6762]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.449978  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8155990eebd84fcf9ebc357e186d1bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.453899  1244 rpcz_store.cc:276] 0628 07:52:22.537905 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:22.542082 (+  4177us) service_pool.cc:225] Handling call
0628 07:52:25.202788 (+2660706us) ts_tablet_manager.cc:1145] Deleting tablet 482095693aac476faf997ea8e1a4b460
0628 07:52:25.206587 (+  3799us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.374486 (+167899us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.400806 (+ 26320us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.426763 (+ 25957us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":47682,"tablet-delete.queue_time_us":2642837}
I20250628 07:52:25.456856  1623 raft_consensus.cc:2747] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.457285  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 (DEFAULT_TABLE table=default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.457701  1244 tablet_replica.cc:331] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:25.458176  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e3e0e52b4924e67904d3acfa4dd5752. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.458357  1244 raft_consensus.cc:2241] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.458084  1229 consensus_queue.cc:260] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.458717  1230 ts_tablet_manager.cc:1428] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.030s	user 0.006s	sys 0.000s
I20250628 07:52:25.458796  1244 raft_consensus.cc:2270] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.460887  1244 ts_tablet_manager.cc:1905] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.465147   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_8dd91f20a94d4b7fa01cb1a8b96a48e0 [id=a9599116e3c04051a9484736a07d350f]). Expected version 1 got 0
I20250628 07:52:25.471452  1230 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.473865  1234 tablet_bootstrap.cc:492] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:25.474357  1234 ts_tablet_manager.cc:1397] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.094s	user 0.012s	sys 0.002s
I20250628 07:52:25.477037  1230 tablet_bootstrap.cc:654] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.479720  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:25.492425  1101 tablet_service.cc:1468] Processing CreateTablet for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (DEFAULT_TABLE table=default.Table_b68d37d5ffd14e17820af67d5f0d7a71 [id=d33855cf2b994e0581978cc1922d6960]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.493816  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fceb8ef4ddb44e7eb11be71a5087bd27. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.494132  1234 raft_consensus.cc:357] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.495078  1640 ts_tablet_manager.cc:1918] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.501546  1230 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.509930  1230 ts_tablet_manager.cc:1397] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.039s	user 0.011s	sys 0.001s
I20250628 07:52:25.509977  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 0b251095244548629ae93d52e6c80ccc (DEFAULT_TABLE table=default.Table_cce67d1f9bbb4434a6e80abe70297d1c [id=9af4a5a7e8aa43d6aa039ad94841acb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.511008  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0b251095244548629ae93d52e6c80ccc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.512336  1230 raft_consensus.cc:357] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.512720  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 5cacde13bc29458da81f29fb4fa2945a (DEFAULT_TABLE table=default.Table_408f187fe62742609a155742ab328fa6 [id=93f9610f0fac41f3b8f407a3efd9b844]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.512882   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 29a77cd9dc244d239a518e004c49e3c8
I20250628 07:52:25.513000  1230 raft_consensus.cc:383] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.513284  1230 raft_consensus.cc:738] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.513639  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cacde13bc29458da81f29fb4fa2945a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.513919  1230 consensus_queue.cc:260] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.488982  1229 ts_tablet_manager.cc:1428] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.053s	user 0.006s	sys 0.001s
I20250628 07:52:25.516170  1230 ts_tablet_manager.cc:1428] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20250628 07:52:25.516760  1229 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:25.517182  1640 log.cc:1199] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:25.517943  1640 ts_tablet_manager.cc:1939] T 01feb8ef40dc4707bd6251fc2d87f46f P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:25.518492  1234 raft_consensus.cc:383] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.518765  1234 raft_consensus.cc:738] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.519402  1234 consensus_queue.cc:260] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.524973  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:25.525740  1230 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.526782   960 tablet_service.cc:1468] Processing CreateTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 (DEFAULT_TABLE table=default.Table_73dcf73d62f245089f066d28320dae9c [id=ab01e12b48e047828205036a96ce6762]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.527808   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8155990eebd84fcf9ebc357e186d1bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.533795  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 185) took 1204 ms. Trace:
I20250628 07:52:25.534096  1640 rpcz_store.cc:276] 0628 07:52:24.329701 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.329877 (+   176us) service_pool.cc:225] Handling call
0628 07:52:25.346550 (+1016673us) ts_tablet_manager.cc:1145] Deleting tablet 01feb8ef40dc4707bd6251fc2d87f46f
0628 07:52:25.346594 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.408227 (+ 61633us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.494789 (+ 86562us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.533741 (+ 38952us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1015908}
I20250628 07:52:25.538398   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 01feb8ef40dc4707bd6251fc2d87f46f (table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]) successfully deleted
I20250628 07:52:25.544566  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 186) took 1190 ms. Trace:
I20250628 07:52:25.544900  1640 rpcz_store.cc:276] 0628 07:52:24.354595 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.354727 (+   132us) service_pool.cc:225] Handling call
0628 07:52:25.534931 (+1180204us) ts_tablet_manager.cc:1145] Deleting tablet 83ffa65248fc4db3aecc5f91b648b83f
0628 07:52:25.541152 (+  6221us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.544447 (+  3295us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1179663}
I20250628 07:52:25.546339  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 187) took 1185 ms. Trace:
W20250628 07:52:25.546865   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 83ffa65248fc4db3aecc5f91b648b83f already in progress: creating tablet
I20250628 07:52:25.548660   801 tablet_service.cc:1468] Processing CreateTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 (DEFAULT_TABLE table=default.Table_73dcf73d62f245089f066d28320dae9c [id=ab01e12b48e047828205036a96ce6762]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.549639   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8155990eebd84fcf9ebc357e186d1bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.556517  1234 ts_tablet_manager.cc:1428] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.078s	user 0.003s	sys 0.001s
I20250628 07:52:25.563169  1234 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:25.568646   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 4e9d585d5f0c4469828561a7df4fb917
I20250628 07:52:25.546604  1640 rpcz_store.cc:276] 0628 07:52:24.360531 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.360637 (+   106us) service_pool.cc:225] Handling call
0628 07:52:25.546063 (+1185426us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:25.546100 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.546294 (+   194us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1184966}
I20250628 07:52:25.570859  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:25.571460   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet ee03996105074308bda56d36ba3a1c5d already in progress: creating tablet
I20250628 07:52:25.575131   804 tablet_service.cc:1468] Processing CreateTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 (DEFAULT_TABLE table=default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.576398   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e3e0e52b4924e67904d3acfa4dd5752. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.577037   799 tablet_service.cc:1468] Processing CreateTablet for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (DEFAULT_TABLE table=default.Table_b68d37d5ffd14e17820af67d5f0d7a71 [id=d33855cf2b994e0581978cc1922d6960]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.577915   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fceb8ef4ddb44e7eb11be71a5087bd27. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.577703   966 tablet_service.cc:1468] Processing CreateTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 (DEFAULT_TABLE table=default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.577706   965 tablet_service.cc:1468] Processing CreateTablet for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (DEFAULT_TABLE table=default.Table_b68d37d5ffd14e17820af67d5f0d7a71 [id=d33855cf2b994e0581978cc1922d6960]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.578763   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e3e0e52b4924e67904d3acfa4dd5752. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.579267   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fceb8ef4ddb44e7eb11be71a5087bd27. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.596778  1640 tablet_replica.cc:331] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:25.597554  1640 raft_consensus.cc:2241] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.597978  1640 raft_consensus.cc:2270] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.599102   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:25.607218  1640 ts_tablet_manager.cc:1905] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.609692   544 catalog_manager.cc:5582] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:25.614637  1230 tablet_bootstrap.cc:654] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.637218  1234 tablet_bootstrap.cc:654] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.641528  1229 tablet_bootstrap.cc:654] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.642601   966 tablet_service.cc:1468] Processing CreateTablet for tablet 0b251095244548629ae93d52e6c80ccc (DEFAULT_TABLE table=default.Table_cce67d1f9bbb4434a6e80abe70297d1c [id=9af4a5a7e8aa43d6aa039ad94841acb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.643815   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0b251095244548629ae93d52e6c80ccc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.652468   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:25.652324   800 tablet_service.cc:1468] Processing CreateTablet for tablet 5cacde13bc29458da81f29fb4fa2945a (DEFAULT_TABLE table=default.Table_408f187fe62742609a155742ab328fa6 [id=93f9610f0fac41f3b8f407a3efd9b844]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.653380   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cacde13bc29458da81f29fb4fa2945a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.658411   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:25.659317  1229 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:25.659705  1229 ts_tablet_manager.cc:1397] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.143s	user 0.013s	sys 0.000s
I20250628 07:52:25.661266  1801 raft_consensus.cc:491] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.661707  1801 raft_consensus.cc:513] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.662513  1229 raft_consensus.cc:357] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.663100  1229 raft_consensus.cc:383] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.663339  1229 raft_consensus.cc:738] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.663488  1801 leader_election.cc:290] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:25.664018  1229 consensus_queue.cc:260] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.666129   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:25.667678  1062 leader_election.cc:343] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.668198  1229 ts_tablet_manager.cc:1428] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.007s	sys 0.000s
I20250628 07:52:25.669624   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:25.671263  1062 leader_election.cc:343] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.671612  1062 leader_election.cc:304] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.673611  1801 raft_consensus.cc:2747] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.674362  1229 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:25.677891   965 tablet_service.cc:1468] Processing CreateTablet for tablet 5cacde13bc29458da81f29fb4fa2945a (DEFAULT_TABLE table=default.Table_408f187fe62742609a155742ab328fa6 [id=93f9610f0fac41f3b8f407a3efd9b844]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.678932   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cacde13bc29458da81f29fb4fa2945a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.680784  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 8226c926201040c993804d2ae8450202 (DEFAULT_TABLE table=default.Table_bda937abf44b485aa9fd46ab195442f8 [id=2a211e6249e24fc89a60604f187a5554]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.681736  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8226c926201040c993804d2ae8450202. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.683118  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 86591ef37c8547189ff8ef270e9bfa9a (DEFAULT_TABLE table=default.Table_bc0991297d684c8fb6295c5ed29cd3c1 [id=0bcb503dea9b4b9b96ae9a3f6bb9f9a5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.684082  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86591ef37c8547189ff8ef270e9bfa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.692042  1229 tablet_bootstrap.cc:654] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.694360  1234 tablet_bootstrap.cc:492] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:25.703058  1234 ts_tablet_manager.cc:1397] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.140s	user 0.009s	sys 0.001s
I20250628 07:52:25.705487  1234 raft_consensus.cc:357] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.699513  1640 ts_tablet_manager.cc:1918] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.706938  1640 log.cc:1199] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:25.708046  1640 ts_tablet_manager.cc:1939] T ef060774191146168ee8d1e13c9cd71b P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:25.702791  1230 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.709301  1230 ts_tablet_manager.cc:1397] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.184s	user 0.006s	sys 0.004s
I20250628 07:52:25.711200  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 191) took 1287 ms. Trace:
I20250628 07:52:25.711377  1229 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:25.711759  1229 ts_tablet_manager.cc:1397] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.043s	user 0.006s	sys 0.003s
I20250628 07:52:25.713352  1640 rpcz_store.cc:276] 0628 07:52:24.424158 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.424268 (+   110us) service_pool.cc:225] Handling call
0628 07:52:25.596613 (+1172345us) ts_tablet_manager.cc:1145] Deleting tablet ef060774191146168ee8d1e13c9cd71b
0628 07:52:25.596671 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.654700 (+ 58029us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.699308 (+ 44608us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.711132 (+ 11824us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1171811}
I20250628 07:52:25.714054  1229 raft_consensus.cc:357] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.714771  1229 raft_consensus.cc:383] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.715024  1229 raft_consensus.cc:738] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.715806  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 193) took 1205 ms. Trace:
I20250628 07:52:25.715613  1229 consensus_queue.cc:260] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.716882  1640 rpcz_store.cc:276] 0628 07:52:24.509995 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.510942 (+   947us) service_pool.cc:225] Handling call
0628 07:52:25.715544 (+1204602us) ts_tablet_manager.cc:1145] Deleting tablet fab31b073e2e49318b4118851f81b6d2
0628 07:52:25.715586 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.715756 (+   170us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1204034}
I20250628 07:52:25.660329   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_09dfad9563454a0e9624d7e15961a374"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.719556  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 200) took 1022 ms. Trace:
W20250628 07:52:25.720319   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_09dfad9563454a0e9624d7e15961a374 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.720414  1640 rpcz_store.cc:276] 0628 07:52:24.697534 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.697663 (+   129us) service_pool.cc:225] Handling call
0628 07:52:25.718758 (+1021095us) ts_tablet_manager.cc:1145] Deleting tablet 9579dd44e5e241aea2671be28e16ef7c
0628 07:52:25.718794 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.719510 (+   716us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":994603}
I20250628 07:52:25.722350  1640 tablet_replica.cc:331] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:25.661588   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b66cef7061354731bda56976a11f758d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.724344  1640 raft_consensus.cc:2241] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.724768  1640 raft_consensus.cc:2270] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:25.725036   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b66cef7061354731bda56976a11f758d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.727906  1640 ts_tablet_manager.cc:1905] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.662930   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_f2b450a384bc4c03abafa265e98da078" } new_table_name: "default.Table_b0c733c64d5449ea86e9103a47c2f578" modify_external_catalogs: true
I20250628 07:52:25.663887   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5d099833475e4b44b3fda7923e027a37"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:25.711555  1230 raft_consensus.cc:357] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.715268   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet ef060774191146168ee8d1e13c9cd71b (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
I20250628 07:52:25.730249  1230 raft_consensus.cc:383] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.730495  1230 raft_consensus.cc:738] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:25.730741   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fab31b073e2e49318b4118851f81b6d2 because the tablet was not found. No further retry: Not found: Tablet not found: fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:25.731180  1230 consensus_queue.cc:260] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:25.731552   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9579dd44e5e241aea2671be28e16ef7c already in progress: creating tablet
I20250628 07:52:25.733292  1230 ts_tablet_manager.cc:1428] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.024s	user 0.006s	sys 0.000s
W20250628 07:52:25.733240   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5d099833475e4b44b3fda7923e027a37 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.734153  1230 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:25.738495   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da]). Expected version 1 got 0
I20250628 07:52:25.739406   544 catalog_manager.cc:5582] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:25.744503   542 master_service.cc:948] ReplaceTablet: received request to replace tablet 01feb8ef40dc4707bd6251fc2d87f46f from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:25.744999  1230 tablet_bootstrap.cc:654] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.757043   539 master_service.cc:948] ReplaceTablet: received request to replace tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:25.759164   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 482095693aac476faf997ea8e1a4b460 (table default.Table_62272d797cf643d295305387e17a746b [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:25.762226   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:25.765214  1106 tablet_service.cc:1468] Processing CreateTablet for tablet f845f32699044867af6f0f2b655cad8e (DEFAULT_TABLE table=default.Table_92144695519f4cc9a21fedc487712d03 [id=77ccc4338ec448b6b07eace3de191874]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.765808  1230 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:25.766197  1230 ts_tablet_manager.cc:1397] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.000s
I20250628 07:52:25.766297  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f845f32699044867af6f0f2b655cad8e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.696538  1244 ts_tablet_manager.cc:1918] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:25.768031  1244 log.cc:1199] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/5e1b0dbbc21b445d8115d963f6aba2a1
I20250628 07:52:25.708410   804 tablet_service.cc:1468] Processing CreateTablet for tablet 0b251095244548629ae93d52e6c80ccc (DEFAULT_TABLE table=default.Table_cce67d1f9bbb4434a6e80abe70297d1c [id=9af4a5a7e8aa43d6aa039ad94841acb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.768334  1230 raft_consensus.cc:357] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.768781  1244 ts_tablet_manager.cc:1939] T 5e1b0dbbc21b445d8115d963f6aba2a1 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:25.763828   799 tablet_service.cc:1515] Processing DeleteTablet for tablet 4e9d585d5f0c4469828561a7df4fb917 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:25.769559   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0b251095244548629ae93d52e6c80ccc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.770767  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 167) took 2107 ms. Trace:
I20250628 07:52:25.771068  1244 rpcz_store.cc:276] 0628 07:52:23.663350 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.663647 (+   297us) service_pool.cc:225] Handling call
0628 07:52:25.457514 (+1793867us) ts_tablet_manager.cc:1145] Deleting tablet 5e1b0dbbc21b445d8115d963f6aba2a1
0628 07:52:25.457573 (+    59us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.539039 (+ 81466us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.696293 (+157254us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.770711 (+ 74418us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1793058}
I20250628 07:52:25.772346  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 169) took 2108 ms. Trace:
I20250628 07:52:25.772626  1244 rpcz_store.cc:276] 0628 07:52:23.663984 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.664169 (+   185us) service_pool.cc:225] Handling call
0628 07:52:25.772091 (+2107922us) ts_tablet_manager.cc:1145] Deleting tablet 2b51d6076e344c8f8e388b77b4b23928
0628 07:52:25.772132 (+    41us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.772295 (+   163us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2107248}
I20250628 07:52:25.773542  1244 tablet_replica.cc:331] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:25.773958   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 16a8bee148664e948529d32c0cf2c793 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:25.773952  1640 ts_tablet_manager.cc:1918] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:25.774240  1244 raft_consensus.cc:2241] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:25.774312  1640 log.cc:1199] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:25.774689  1244 raft_consensus.cc:2270] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.774899  1640 ts_tablet_manager.cc:1939] T 0f29c8ffb7df410295278eee9c578b03 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:25.776353  1244 ts_tablet_manager.cc:1905] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.777176  1234 raft_consensus.cc:383] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.768935  1230 raft_consensus.cc:383] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:25.778179  1230 raft_consensus.cc:738] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.777498  1234 raft_consensus.cc:738] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:25.778980  1234 consensus_queue.cc:260] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.779532  1230 consensus_queue.cc:260] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.741021   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:25.784080  1807 tablet_replica.cc:331] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:25.784839  1807 raft_consensus.cc:2241] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:25.785715  1807 raft_consensus.cc:2270] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:25.787976  1807 ts_tablet_manager.cc:1905] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:25.790705   799 tablet_service.cc:1468] Processing CreateTablet for tablet 8226c926201040c993804d2ae8450202 (DEFAULT_TABLE table=default.Table_bda937abf44b485aa9fd46ab195442f8 [id=2a211e6249e24fc89a60604f187a5554]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.791610   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8226c926201040c993804d2ae8450202. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.794299   966 tablet_service.cc:1468] Processing CreateTablet for tablet 86591ef37c8547189ff8ef270e9bfa9a (DEFAULT_TABLE table=default.Table_bc0991297d684c8fb6295c5ed29cd3c1 [id=0bcb503dea9b4b9b96ae9a3f6bb9f9a5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.794961   798 tablet_service.cc:1468] Processing CreateTablet for tablet 86591ef37c8547189ff8ef270e9bfa9a (DEFAULT_TABLE table=default.Table_bc0991297d684c8fb6295c5ed29cd3c1 [id=0bcb503dea9b4b9b96ae9a3f6bb9f9a5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.796218   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86591ef37c8547189ff8ef270e9bfa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.796504   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86591ef37c8547189ff8ef270e9bfa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.797369  1785 raft_consensus.cc:491] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.797770  1785 raft_consensus.cc:513] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.799489  1785 leader_election.cc:290] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:25.800673   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:25.813699   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet f9a553c69c254fdcaf413b53018bbfb2 has no consensus state
I20250628 07:52:25.814535   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:17 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:25.815712   960 tablet_service.cc:1468] Processing CreateTablet for tablet 8226c926201040c993804d2ae8450202 (DEFAULT_TABLE table=default.Table_bda937abf44b485aa9fd46ab195442f8 [id=2a211e6249e24fc89a60604f187a5554]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.817219  1234 ts_tablet_manager.cc:1428] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.114s	user 0.005s	sys 0.000s
I20250628 07:52:25.822739  1229 ts_tablet_manager.cc:1428] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.111s	user 0.006s	sys 0.001s
I20250628 07:52:25.823629  1229 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:25.824330  1234 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:25.828137   962 tablet_service.cc:1468] Processing CreateTablet for tablet 7d2a96449ed44d76a015e929d6a6edb0 (DEFAULT_TABLE table=default.Table_5baf5e16d9f6482ab94b15a2706e3fbd [id=77ef1587831e44bd9a93a373b6fd9d0f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.828370  1782 consensus_queue.cc:1035] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:25.833459   963 tablet_service.cc:1468] Processing CreateTablet for tablet f845f32699044867af6f0f2b655cad8e (DEFAULT_TABLE table=default.Table_92144695519f4cc9a21fedc487712d03 [id=77ccc4338ec448b6b07eace3de191874]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.831259   801 tablet_service.cc:1468] Processing CreateTablet for tablet 7d2a96449ed44d76a015e929d6a6edb0 (DEFAULT_TABLE table=default.Table_5baf5e16d9f6482ab94b15a2706e3fbd [id=77ef1587831e44bd9a93a373b6fd9d0f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.836926   800 tablet_service.cc:1468] Processing CreateTablet for tablet f845f32699044867af6f0f2b655cad8e (DEFAULT_TABLE table=default.Table_92144695519f4cc9a21fedc487712d03 [id=77ccc4338ec448b6b07eace3de191874]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.840497   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7d2a96449ed44d76a015e929d6a6edb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.845188   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f845f32699044867af6f0f2b655cad8e. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:25.848170   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 29a77cd9dc244d239a518e004c49e3c8 because the tablet was not found. No further retry: Not found: Tablet not found: 29a77cd9dc244d239a518e004c49e3c8
I20250628 07:52:25.787400  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 7d2a96449ed44d76a015e929d6a6edb0 (DEFAULT_TABLE table=default.Table_5baf5e16d9f6482ab94b15a2706e3fbd [id=77ef1587831e44bd9a93a373b6fd9d0f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.800484  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "60179e6765b145adbf637a8435eef724" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:25.850375  1125 raft_consensus.cc:2391] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:25.851642   758 leader_election.cc:304] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.853034  1785 raft_consensus.cc:3058] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:25.861482  1785 raft_consensus.cc:2747] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.863799  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7d2a96449ed44d76a015e929d6a6edb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.877990   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:25.881071   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:25.881141  1102 tablet_service.cc:1468] Processing CreateTablet for tablet a447edf0960c4739930450a8c24ff6f7 (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.892035   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_96f7aea7a2b445139de2bf323b687f73" } new_table_name: "default.Table_46b78ac794754d8da31f4b39fb3f8fca" modify_external_catalogs: true
I20250628 07:52:25.895184  1234 tablet_bootstrap.cc:654] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:25.912513   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5a2877b9c29b49c18d0909b99b08fa80" } modify_external_catalogs: true
I20250628 07:52:25.913097  1230 ts_tablet_manager.cc:1428] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.147s	user 0.008s	sys 0.000s
I20250628 07:52:25.913143   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5a2877b9c29b49c18d0909b99b08fa80" } modify_external_catalogs: true
W20250628 07:52:25.914330   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet fef70db0996f48c69574337f8f7ecf37 consensus state has no leader
I20250628 07:52:25.919721   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7d2a96449ed44d76a015e929d6a6edb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.921335   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f845f32699044867af6f0f2b655cad8e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.922080  1782 consensus_queue.cc:1035] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:25.923596  1801 raft_consensus.cc:491] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.924062  1801 raft_consensus.cc:513] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.926229  1801 leader_election.cc:290] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:25.926846   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:25.928575  1062 leader_election.cc:343] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.928725   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:25.930757  1062 leader_election.cc:343] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.931082  1062 leader_election.cc:304] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:25.933758  1801 raft_consensus.cc:2747] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:25.938234  1782 raft_consensus.cc:491] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.938652  1782 raft_consensus.cc:513] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.940392  1782 leader_election.cc:290] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:25.941212   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:25.941377  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:25.941946  1128 raft_consensus.cc:2466] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:25.942225   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_32928f5c3c044deaa68ad9b7971a3aaf"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:25.942508   920 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:25.943017   921 leader_election.cc:304] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:25.943678   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_32928f5c3c044deaa68ad9b7971a3aaf in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:25.944169  1782 raft_consensus.cc:2802] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:25.944608  1782 raft_consensus.cc:491] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:25.945050  1782 raft_consensus.cc:3058] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:25.948158   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:25.948208   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8226c926201040c993804d2ae8450202. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.949609  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 (DEFAULT_TABLE table=default.Table_026074aa331e455ebaaf270237f94590 [id=a4c74a0d7c804bbbb875dd4ee2607710]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:25.884650  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a447edf0960c4739930450a8c24ff6f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.966531  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29a77cd9dc244d239a518e004c49e3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:25.972345  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:25.973527  1819 raft_consensus.cc:491] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:25.973994  1819 raft_consensus.cc:513] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:25.975889  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 201) took 1263 ms. Trace:
I20250628 07:52:25.975762  1819 leader_election.cc:290] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:25.976194  1640 rpcz_store.cc:276] 0628 07:52:24.712508 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.758149 (+ 45641us) service_pool.cc:225] Handling call
0628 07:52:25.722208 (+964059us) ts_tablet_manager.cc:1145] Deleting tablet 0f29c8ffb7df410295278eee9c578b03
0628 07:52:25.722245 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.751748 (+ 29503us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.773728 (+ 21980us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:25.975836 (+202108us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":953316}
I20250628 07:52:25.977219  1640 tablet_replica.cc:331] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:25.978317   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 0f29c8ffb7df410295278eee9c578b03 (table default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]) successfully deleted
I20250628 07:52:25.978348  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:25.978935  1128 raft_consensus.cc:2466] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:25.980036   921 leader_election.cc:304] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:25.983086  1230 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:25.985064   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 48253.6us
I20250628 07:52:25.988270  1817 raft_consensus.cc:2802] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:25.991585   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:25.992332  1234 tablet_bootstrap.cc:492] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:25.992754  1234 ts_tablet_manager.cc:1397] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.169s	user 0.008s	sys 0.002s
I20250628 07:52:25.998234   824 raft_consensus.cc:3058] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:25.999785  1234 raft_consensus.cc:357] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.000497  1234 raft_consensus.cc:383] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.000823  1234 raft_consensus.cc:738] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.001523  1234 consensus_queue.cc:260] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.012571  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.013571  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.019088   800 tablet_service.cc:1468] Processing CreateTablet for tablet a447edf0960c4739930450a8c24ff6f7 (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:25.993767   920 leader_election.cc:343] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.020138   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a447edf0960c4739930450a8c24ff6f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.020591   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:26.020605  1779 consensus_queue.cc:1035] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:26.021224   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da]). Expected version 1 got 0
I20250628 07:52:26.023005  1817 raft_consensus.cc:491] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:26.023339  1817 raft_consensus.cc:3058] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.026938  1234 ts_tablet_manager.cc:1428] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.034s	user 0.004s	sys 0.000s
I20250628 07:52:26.026953   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 5e1b0dbbc21b445d8115d963f6aba2a1 (table default.Table_218e7d52d78541a0b85a22af790324a6 [id=55e5becb6db2459c94335b21d962e159]) successfully deleted
W20250628 07:52:26.027511   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2b51d6076e344c8f8e388b77b4b23928 because the tablet was not found. No further retry: Not found: Tablet not found: 2b51d6076e344c8f8e388b77b4b23928
I20250628 07:52:26.030321   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f with delete_type TABLET_DATA_DELETED (replaced by tablet a447edf0960c4739930450a8c24ff6f7) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.030721  1234 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:26.030915   959 tablet_service.cc:1515] Processing DeleteTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.030915   961 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.032747   961 tablet_service.cc:1468] Processing CreateTablet for tablet a447edf0960c4739930450a8c24ff6f7 (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:26.034265   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 50629.2us
I20250628 07:52:26.043009  1244 ts_tablet_manager.cc:1918] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:26.044067  1244 log.cc:1199] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/d62ecf32bcc54310b9114d5655be7673
I20250628 07:52:26.044819  1244 ts_tablet_manager.cc:1939] T d62ecf32bcc54310b9114d5655be7673 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:26.045639  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 170) took 2381 ms. Trace:
I20250628 07:52:26.046067   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a447edf0960c4739930450a8c24ff6f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.047641  1821 consensus_queue.cc:1035] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:26.057888  1782 raft_consensus.cc:513] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.059919  1782 leader_election.cc:290] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.060737  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:26.061300  1128 raft_consensus.cc:3058] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.061426  1640 raft_consensus.cc:2241] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.061199  1244 rpcz_store.cc:276] 0628 07:52:23.664336 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.664514 (+   178us) service_pool.cc:225] Handling call
0628 07:52:25.773410 (+2108896us) ts_tablet_manager.cc:1145] Deleting tablet d62ecf32bcc54310b9114d5655be7673
0628 07:52:25.773448 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:25.941505 (+168057us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.042779 (+101274us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.045586 (+  2807us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2108271}
I20250628 07:52:26.061929  1640 raft_consensus.cc:2270] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.063560  1640 ts_tablet_manager.cc:1905] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.073447   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:26.074692   920 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:26.076035   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_46b78ac794754d8da31f4b39fb3f8fca [id=1a365914a60944d095fa09775cc75aa1], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet fef70db0996f48c69574337f8f7ecf37 consensus state has no leader
I20250628 07:52:26.077651  1128 raft_consensus.cc:2466] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
W20250628 07:52:26.078518   758 consensus_peers.cc:489] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:26.009910  1229 tablet_bootstrap.cc:654] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.079321   921 leader_election.cc:304] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:26.079922  1817 raft_consensus.cc:513] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.083421  1817 leader_election.cc:290] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.083295   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:26.085599  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "532ba24d0d644313b9ef4e3ec10b0fed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:26.086108  1128 raft_consensus.cc:3058] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.086398  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 1ed24dbf9c234b43b352114d5219bbea (DEFAULT_TABLE table=default.Table_88c945c579fe4aaf9c7acecfc6b88288 [id=f9005880df6b41aa86533a453b679f3d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.087325  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1ed24dbf9c234b43b352114d5219bbea. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.090421  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 168) took 2426 ms. Trace:
I20250628 07:52:26.091502   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:26.095918  1244 rpcz_store.cc:276] 0628 07:52:23.663656 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.668427 (+  4771us) service_pool.cc:225] Handling call
0628 07:52:26.090168 (+2421741us) ts_tablet_manager.cc:1145] Deleting tablet 7a3e80af50b542ce94ea3cb2b2cef3b8
0628 07:52:26.090213 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.090351 (+   138us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2421151}
W20250628 07:52:26.084924   920 leader_election.cc:343] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.105769  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 173) took 2402 ms. Trace:
I20250628 07:52:26.106300  1128 raft_consensus.cc:2466] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:26.107259   921 leader_election.cc:304] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:26.108249  1779 raft_consensus.cc:2802] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:26.108673  1779 raft_consensus.cc:695] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:26.109755   965 tablet_service.cc:1468] Processing CreateTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 (DEFAULT_TABLE table=default.Table_026074aa331e455ebaaf270237f94590 [id=a4c74a0d7c804bbbb875dd4ee2607710]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.110751   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29a77cd9dc244d239a518e004c49e3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.111351   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:26.080474  1818 raft_consensus.cc:2802] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:26.113854  1818 raft_consensus.cc:695] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:26.115463  1818 consensus_queue.cc:237] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.116338  1821 raft_consensus.cc:491] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.116742  1821 raft_consensus.cc:513] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.118552  1821 leader_election.cc:290] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.109517  1779 consensus_queue.cc:237] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:26.123842   921 leader_election.cc:343] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: e4e02676d24a4345908dbbc42fc9c1ab
I20250628 07:52:26.124643   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e4e02676d24a4345908dbbc42fc9c1ab" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:26.125954   920 leader_election.cc:343] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.126365   920 leader_election.cc:304] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:26.128254  1817 raft_consensus.cc:2747] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.130219  1234 tablet_bootstrap.cc:654] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.135005  1807 ts_tablet_manager.cc:1918] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:26.122210  1244 rpcz_store.cc:276] 0628 07:52:23.701321 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.701432 (+   111us) service_pool.cc:225] Handling call
0628 07:52:26.102955 (+2401523us) ts_tablet_manager.cc:1145] Deleting tablet 0dffb3eb778946869b914e5b05a67a69
0628 07:52:26.102994 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.103253 (+   259us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2400952}
I20250628 07:52:26.135618  1807 log.cc:1199] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/4e9d585d5f0c4469828561a7df4fb917
I20250628 07:52:26.136014  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 175) took 2433 ms. Trace:
I20250628 07:52:26.136353  1244 rpcz_store.cc:276] 0628 07:52:23.702644 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.702750 (+   106us) service_pool.cc:225] Handling call
0628 07:52:26.135671 (+2432921us) ts_tablet_manager.cc:1145] Deleting tablet 5ec8c1d9386a4922b55df5e06039858d
0628 07:52:26.135710 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.135961 (+   251us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2432429}
I20250628 07:52:26.137506  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 177) took 2243 ms. Trace:
I20250628 07:52:26.137851  1244 rpcz_store.cc:276] 0628 07:52:23.893755 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.894839 (+  1084us) service_pool.cc:225] Handling call
0628 07:52:26.137206 (+2242367us) ts_tablet_manager.cc:1145] Deleting tablet 9a248bf8a62b4b4eb206c0daff5e0568
0628 07:52:26.137241 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.137458 (+   217us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2229313}
I20250628 07:52:26.150790  1230 tablet_bootstrap.cc:654] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.138682   801 tablet_service.cc:1468] Processing CreateTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 (DEFAULT_TABLE table=default.Table_026074aa331e455ebaaf270237f94590 [id=a4c74a0d7c804bbbb875dd4ee2607710]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.139822   804 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:26.142560   798 tablet_service.cc:1468] Processing CreateTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.149381   818 raft_consensus.cc:3058] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.162639   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:26.171290   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29a77cd9dc244d239a518e004c49e3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.180023   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.156680  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 178) took 2229 ms. Trace:
I20250628 07:52:26.157343  1234 tablet_bootstrap.cc:492] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:26.186587  1234 ts_tablet_manager.cc:1397] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.156s	user 0.006s	sys 0.004s
I20250628 07:52:26.186398  1244 rpcz_store.cc:276] 0628 07:52:23.927039 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:23.937206 (+ 10167us) service_pool.cc:225] Handling call
0628 07:52:26.156298 (+2219092us) ts_tablet_manager.cc:1145] Deleting tablet b51b51eb951d45a78bd2bffe47834b95
0628 07:52:26.156365 (+    67us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.156622 (+   257us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":7424,"tablet-delete.queue_time_us":2218599}
I20250628 07:52:26.189000  1234 raft_consensus.cc:357] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.190438  1234 raft_consensus.cc:383] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.190763  1234 raft_consensus.cc:738] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.191437  1234 consensus_queue.cc:260] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.194039  1234 ts_tablet_manager.cc:1428] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.007s	user 0.005s	sys 0.000s
I20250628 07:52:26.200037   544 catalog_manager.cc:5582] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:26.189303  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 183) took 1810 ms. Trace:
I20250628 07:52:26.201135  1244 rpcz_store.cc:276] 0628 07:52:24.379332 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.379439 (+   107us) service_pool.cc:225] Handling call
0628 07:52:26.188954 (+1809515us) ts_tablet_manager.cc:1145] Deleting tablet 83ffa65248fc4db3aecc5f91b648b83f
0628 07:52:26.188998 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.189252 (+   254us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1809042}
I20250628 07:52:26.203830  1234 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:26.205447  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 184) took 1825 ms. Trace:
I20250628 07:52:26.208390   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:26.208853  1244 rpcz_store.cc:276] 0628 07:52:24.380189 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.380272 (+    83us) service_pool.cc:225] Handling call
0628 07:52:26.202188 (+1821916us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:26.205156 (+  2968us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.205397 (+   241us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1821516}
I20250628 07:52:26.211408   544 catalog_manager.cc:5582] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:26.140592   966 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.144124   959 tablet_service.cc:1468] Processing CreateTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.214107   959 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.215370   544 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:26.147353  1229 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:26.172024  1782 consensus_queue.cc:1035] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:26.192638  1818 raft_consensus.cc:491] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.225485  1818 raft_consensus.cc:513] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.227178  1229 ts_tablet_manager.cc:1397] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.404s	user 0.020s	sys 0.001s
I20250628 07:52:26.227303  1818 leader_election.cc:290] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.230624  1807 ts_tablet_manager.cc:1939] T 4e9d585d5f0c4469828561a7df4fb917 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:26.283389  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 182) took 1904 ms. Trace:
I20250628 07:52:26.283727  1244 rpcz_store.cc:276] 0628 07:52:24.379067 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.381200 (+  2133us) service_pool.cc:225] Handling call
0628 07:52:26.210897 (+1829697us) ts_tablet_manager.cc:1145] Deleting tablet 01feb8ef40dc4707bd6251fc2d87f46f
0628 07:52:26.210936 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.283322 (+ 72386us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1829164}
I20250628 07:52:26.284633   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_06cb6f2bb04a4517ad00c1c925fe4ef9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:26.286231   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_06cb6f2bb04a4517ad00c1c925fe4ef9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.287880   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_efdb291a75464ca09777b59f822b3ce7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:26.288183   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8dd91f20a94d4b7fa01cb1a8b96a48e0" } new_table_name: "default.Table_22a7acccf75140a8a636cec2a73177f6" modify_external_catalogs: true
W20250628 07:52:26.289371   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_efdb291a75464ca09777b59f822b3ce7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.289799   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_624e66567efa4b4f9ac68e502edbceda"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:26.291115   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_624e66567efa4b4f9ac68e502edbceda in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.292474   543 master_service.cc:948] ReplaceTablet: received request to replace tablet f42b22f7096e46cf9cc09d1caecc9efb from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:26.293030   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_3f52d53f779c43b38b6cd8d7332cd51f" } new_table_name: "default.Table_3a9e65464a9842779a108c92d8bd414e" modify_external_catalogs: true
I20250628 07:52:26.293875  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 187) took 1677 ms. Trace:
I20250628 07:52:26.294217  1244 rpcz_store.cc:276] 0628 07:52:24.616438 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.616547 (+   109us) service_pool.cc:225] Handling call
0628 07:52:26.284515 (+1667968us) ts_tablet_manager.cc:1145] Deleting tablet ef060774191146168ee8d1e13c9cd71b
0628 07:52:26.284553 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.293803 (+  9250us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1666545}
I20250628 07:52:26.295274  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 197) took 1480 ms. Trace:
I20250628 07:52:26.295547  1244 rpcz_store.cc:276] 0628 07:52:24.814610 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.816103 (+  1493us) service_pool.cc:225] Handling call
0628 07:52:26.294979 (+1478876us) ts_tablet_manager.cc:1145] Deleting tablet 9579dd44e5e241aea2671be28e16ef7c
0628 07:52:26.295012 (+    33us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.295230 (+   218us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1468027}
I20250628 07:52:26.296289  1234 tablet_bootstrap.cc:654] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.296396  1244 tablet_replica.cc:331] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:26.297071  1244 raft_consensus.cc:2241] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.301398   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "01feb8ef40dc4707bd6251fc2d87f46f" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:26.304199  1785 raft_consensus.cc:491] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.304594  1785 raft_consensus.cc:513] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.306445  1785 leader_election.cc:290] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:26.247794   921 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 01feb8ef40dc4707bd6251fc2d87f46f
W20250628 07:52:26.306957   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 59210us
I20250628 07:52:26.307951   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_caf24e76219f4823a328d7048ce15d24" } modify_external_catalogs: true
W20250628 07:52:26.309675   920 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.310107   920 leader_election.cc:304] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:26.310195   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fab31b073e2e49318b4118851f81b6d2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:26.310793   983 raft_consensus.cc:2466] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:26.314785   757 leader_election.cc:304] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:26.316011  1818 raft_consensus.cc:2747] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.317759  1229 raft_consensus.cc:357] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.318413  1229 raft_consensus.cc:383] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.318760  1229 raft_consensus.cc:738] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.319006  1244 raft_consensus.cc:2270] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.319386  1229 consensus_queue.cc:260] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.327181   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:26.328171   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da]). Expected version 1 got 0
I20250628 07:52:26.330448  1807 tablet_replica.cc:331] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:26.330868   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_caf24e76219f4823a328d7048ce15d24" } modify_external_catalogs: true
W20250628 07:52:26.335410   758 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:26.336450   799 tablet_service.cc:1468] Processing CreateTablet for tablet 1ed24dbf9c234b43b352114d5219bbea (DEFAULT_TABLE table=default.Table_88c945c579fe4aaf9c7acecfc6b88288 [id=f9005880df6b41aa86533a453b679f3d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.336761  1785 raft_consensus.cc:2802] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:26.337133  1785 raft_consensus.cc:491] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:26.337612  1785 raft_consensus.cc:3058] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.353390  1230 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:26.354789  1230 ts_tablet_manager.cc:1397] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.372s	user 0.010s	sys 0.000s
I20250628 07:52:26.337007   961 tablet_service.cc:1468] Processing CreateTablet for tablet 1ed24dbf9c234b43b352114d5219bbea (DEFAULT_TABLE table=default.Table_88c945c579fe4aaf9c7acecfc6b88288 [id=f9005880df6b41aa86533a453b679f3d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.355782   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1ed24dbf9c234b43b352114d5219bbea. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.359792  1230 raft_consensus.cc:357] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.360414  1230 raft_consensus.cc:383] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.360646  1230 raft_consensus.cc:738] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.364054   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1ed24dbf9c234b43b352114d5219bbea. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.366233  1230 consensus_queue.cc:260] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.368677  1230 ts_tablet_manager.cc:1428] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.014s	user 0.004s	sys 0.000s
I20250628 07:52:26.370283  1244 ts_tablet_manager.cc:1905] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:26.372542   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 9ba2a10acdf0462bb05628351cdbbbe4 consensus state has no leader
I20250628 07:52:26.375257   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:26.377127   543 catalog_manager.cc:6652] ReplaceTablet: tablet f42b22f7096e46cf9cc09d1caecc9efb deleted and replaced by tablet e90e850036444335a7e67b383b7e6e4b
I20250628 07:52:26.384846  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb with delete_type TABLET_DATA_DELETED (replaced by tablet e90e850036444335a7e67b383b7e6e4b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:26.387707   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet d62ecf32bcc54310b9114d5655be7673 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
I20250628 07:52:26.400843   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:26.382543  1818 consensus_queue.cc:1035] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:26.386587  1230 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:26.406828  1807 raft_consensus.cc:2241] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.430142  1831 raft_consensus.cc:491] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.430615  1831 raft_consensus.cc:513] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.432410  1831 leader_election.cc:290] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:26.437884   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 50199us
W20250628 07:52:26.438519   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 7a3e80af50b542ce94ea3cb2b2cef3b8 because the tablet was not found. No further retry: Not found: Tablet not found: 7a3e80af50b542ce94ea3cb2b2cef3b8
W20250628 07:52:26.439251   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 0dffb3eb778946869b914e5b05a67a69 already in progress: creating tablet
W20250628 07:52:26.439932   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5ec8c1d9386a4922b55df5e06039858d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5ec8c1d9386a4922b55df5e06039858d already in progress: creating tablet
W20250628 07:52:26.440785   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9a248bf8a62b4b4eb206c0daff5e0568 already in progress: creating tablet
W20250628 07:52:26.442947   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b51b51eb951d45a78bd2bffe47834b95 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b51b51eb951d45a78bd2bffe47834b95 already in progress: creating tablet
W20250628 07:52:26.443462   758 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:26.447947  1785 raft_consensus.cc:513] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.403169  1229 ts_tablet_manager.cc:1428] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.176s	user 0.004s	sys 0.000s
W20250628 07:52:26.441203   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 126511us
W20250628 07:52:26.462697   757 leader_election.cc:343] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:26.463229   757 leader_election.cc:304] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:26.464305   758 leader_election.cc:343] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:26.464982  1831 raft_consensus.cc:2747] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.467558   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fab31b073e2e49318b4118851f81b6d2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:26.468430   983 raft_consensus.cc:3058] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.480278  1838 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:26.428099  1807 raft_consensus.cc:2270] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.496534  1244 ts_tablet_manager.cc:1918] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:26.496987  1244 log.cc:1199] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/29acadb819ef46318e2ba5ba2d3c3c37
I20250628 07:52:26.496752  1837 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:26.498498  1785 leader_election.cc:290] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.499414   983 raft_consensus.cc:2466] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:26.497697  1244 ts_tablet_manager.cc:1939] T 29acadb819ef46318e2ba5ba2d3c3c37 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:26.501964  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 190) took 1790 ms. Trace:
I20250628 07:52:26.497653  1801 raft_consensus.cc:491] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.504881  1801 raft_consensus.cc:513] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.507205  1801 leader_election.cc:290] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:26.510691   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:26.535867  1062 leader_election.cc:343] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.484506  1229 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:26.521054  1818 raft_consensus.cc:491] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.536693  1818 raft_consensus.cc:513] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.539760  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:26.540340  1818 leader_election.cc:290] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.540398  1125 raft_consensus.cc:2466] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:26.504205  1841 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:26.541491   921 leader_election.cc:304] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:26.515501   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:26.543155  1062 leader_election.cc:343] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.543422  1062 leader_election.cc:304] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:26.544358  1801 raft_consensus.cc:2747] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.545109  1818 raft_consensus.cc:2802] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:26.545554  1818 raft_consensus.cc:491] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:26.545917  1818 raft_consensus.cc:3058] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:26.546540  1229 tablet_bootstrap.cc:654] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.530774  1807 ts_tablet_manager.cc:1905] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:26.532985   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 70324.5us
I20250628 07:52:26.551352   757 leader_election.cc:304] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:26.556854  1234 tablet_bootstrap.cc:492] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:26.444492   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:26.549165  1244 rpcz_store.cc:276] 0628 07:52:24.711418 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.725528 (+ 14110us) service_pool.cc:225] Handling call
0628 07:52:26.296265 (+1570737us) ts_tablet_manager.cc:1145] Deleting tablet 29acadb819ef46318e2ba5ba2d3c3c37
0628 07:52:26.296304 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.388911 (+ 92607us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.496328 (+107417us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.501893 (+  5565us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":40678,"tablet-delete.queue_time_us":1469112}
I20250628 07:52:26.557324  1234 ts_tablet_manager.cc:1397] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.354s	user 0.009s	sys 0.019s
I20250628 07:52:26.557484   543 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da]). Expected version 1 got 0
I20250628 07:52:26.465890   540 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet f8628754973246a598c537127ea4b1a8
W20250628 07:52:26.467217   539 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2f8099bd0c9649b492edd8b657fd19e3 consensus state has no leader
W20250628 07:52:26.501461   531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111) [suppressed 6 similar messages]
W20250628 07:52:26.558984   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 116039us
I20250628 07:52:26.556785  1840 raft_consensus.cc:2802] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
W20250628 07:52:26.559540   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 83ffa65248fc4db3aecc5f91b648b83f already in progress: creating tablet
I20250628 07:52:26.559437  1234 raft_consensus.cc:357] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:26.560086   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet ee03996105074308bda56d36ba3a1c5d already in progress: creating tablet
I20250628 07:52:26.560235  1234 raft_consensus.cc:383] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:26.560581   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 01feb8ef40dc4707bd6251fc2d87f46f because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 01feb8ef40dc4707bd6251fc2d87f46f already in progress: creating tablet
I20250628 07:52:26.560528  1234 raft_consensus.cc:738] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.561290   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:26.565568   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da]). Expected version 1 got 0
W20250628 07:52:26.565969   920 leader_election.cc:343] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.566865  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 189) took 1855 ms. Trace:
I20250628 07:52:26.561376  1840 raft_consensus.cc:695] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:26.568655  1840 consensus_queue.cc:237] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.571516  1640 ts_tablet_manager.cc:1918] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:26.571926  1640 log.cc:1199] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/2d77be1f37d2400f842c01019e4eb474
I20250628 07:52:26.572589  1640 ts_tablet_manager.cc:1939] T 2d77be1f37d2400f842c01019e4eb474 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:26.567142  1244 rpcz_store.cc:276] 0628 07:52:24.711070 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.730518 (+ 19448us) service_pool.cc:225] Handling call
0628 07:52:26.558237 (+1827719us) ts_tablet_manager.cc:1145] Deleting tablet d39d14572c4647c7b4880c71a8707c08
0628 07:52:26.566657 (+  8420us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.566807 (+   150us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":583,"tablet-delete.queue_time_us":1691928}
I20250628 07:52:26.573309  1230 tablet_bootstrap.cc:654] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.573802  1244 tablet_replica.cc:331] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:26.574541  1244 raft_consensus.cc:2241] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:26.571707  1818 raft_consensus.cc:513] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.575822  1244 raft_consensus.cc:2270] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:26.576659   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet ef060774191146168ee8d1e13c9cd71b because tablet deleting was already in progress. No further retry: Already present: State transition of tablet ef060774191146168ee8d1e13c9cd71b already in progress: creating tablet
W20250628 07:52:26.577315   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9579dd44e5e241aea2671be28e16ef7c already in progress: creating tablet
I20250628 07:52:26.578279   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 29acadb819ef46318e2ba5ba2d3c3c37 (table default.Table_8c096e0d11db417187a1bde167c0735f [id=b54ee924bae54582b3e5ec4750714dd2]) successfully deleted
W20250628 07:52:26.579061   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet d39d14572c4647c7b4880c71a8707c08 because the tablet was not found. No further retry: Not found: Tablet not found: d39d14572c4647c7b4880c71a8707c08
I20250628 07:52:26.583020   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:26.584339   920 leader_election.cc:343] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.586509  1818 leader_election.cc:290] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:26.587482   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 86027us
I20250628 07:52:26.588130  1244 ts_tablet_manager.cc:1905] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.590162  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:26.590780  1125 raft_consensus.cc:3058] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:26.597218   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20250628 07:52:26.598104  1125 raft_consensus.cc:2466] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:26.599263   921 leader_election.cc:304] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:26.600425  1818 raft_consensus.cc:2802] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:26.600991  1818 raft_consensus.cc:695] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:26.601807  1818 consensus_queue.cc:237] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.611017  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 205) took 1678 ms. Trace:
I20250628 07:52:26.620704   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 4e9d585d5f0c4469828561a7df4fb917 (table default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]) successfully deleted
I20250628 07:52:26.618747  1640 rpcz_store.cc:276] 0628 07:52:24.932536 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.932675 (+   139us) service_pool.cc:225] Handling call
0628 07:52:25.977044 (+1044369us) ts_tablet_manager.cc:1145] Deleting tablet 2d77be1f37d2400f842c01019e4eb474
0628 07:52:25.977107 (+    63us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.429057 (+451950us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.571286 (+142229us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.610959 (+ 39673us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1043860}
I20250628 07:52:26.623234  1801 raft_consensus.cc:491] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.623607   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]) successfully deleted
I20250628 07:52:26.623677  1801 raft_consensus.cc:513] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.624199   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_14e4b6f1c4f344a280e1901e7f772fbd [id=475e75d866ea4a5585227052d39db9da] alter complete (version 1)
I20250628 07:52:26.625522  1801 leader_election.cc:290] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:26.625896   798 tablet_service.cc:1515] Processing DeleteTablet for tablet fab31b073e2e49318b4118851f81b6d2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:26.627089  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 206) took 1463 ms. Trace:
I20250628 07:52:26.627682   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5475a89692eb450d82cde521ec8c9a73" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:26.627640   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5475a89692eb450d82cde521ec8c9a73" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:26.629271  1062 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:26.629957  1062 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.630292  1062 leader_election.cc:304] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:26.632020  1801 raft_consensus.cc:2747] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.632046   798 tablet_service.cc:1515] Processing DeleteTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb with delete_type TABLET_DATA_DELETED (replaced by tablet e90e850036444335a7e67b383b7e6e4b) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:26.630345  1640 rpcz_store.cc:276] 0628 07:52:25.163688 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.163812 (+   124us) service_pool.cc:225] Handling call
0628 07:52:26.626846 (+1463034us) ts_tablet_manager.cc:1145] Deleting tablet 9a248bf8a62b4b4eb206c0daff5e0568
0628 07:52:26.626885 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.627040 (+   155us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1461068}
W20250628 07:52:26.633788   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because the tablet was not found. No further retry: Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:26.610422  1234 consensus_queue.cc:260] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.642763  1234 ts_tablet_manager.cc:1428] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.085s	user 0.002s	sys 0.003s
I20250628 07:52:26.643833   961 tablet_service.cc:1515] Processing DeleteTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb with delete_type TABLET_DATA_DELETED (replaced by tablet e90e850036444335a7e67b383b7e6e4b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.649058   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_92144695519f4cc9a21fedc487712d03" } modify_external_catalogs: true
I20250628 07:52:26.649639   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_92144695519f4cc9a21fedc487712d03" } modify_external_catalogs: true
I20250628 07:52:26.649904  1125 raft_consensus.cc:1273] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20250628 07:52:26.650106   920 consensus_peers.cc:489] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:26.651312  1782 consensus_queue.cc:1035] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:26.653292   543 master_service.cc:948] ReplaceTablet: received request to replace tablet 2ae890710f7f44cf899dd19e96fa9181 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:26.654142  1234 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:26.654879   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5ad5f13abefc445b8c69ab0be547644d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:26.656358   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5ad5f13abefc445b8c69ab0be547644d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.662379  1229 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:26.662786  1229 ts_tablet_manager.cc:1397] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.178s	user 0.012s	sys 0.000s
I20250628 07:52:26.666512   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 01feb8ef40dc4707bd6251fc2d87f46f with delete_type TABLET_DATA_DELETED (replaced by tablet a447edf0960c4739930450a8c24ff6f7) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:26.673470  1230 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:26.673844  1230 ts_tablet_manager.cc:1397] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.287s	user 0.007s	sys 0.007s
I20250628 07:52:26.676218  1230 raft_consensus.cc:357] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.677289  1640 tablet_replica.cc:331] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:26.664986  1229 raft_consensus.cc:357] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.687844  1229 raft_consensus.cc:383] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.688124  1229 raft_consensus.cc:738] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.689319  1640 raft_consensus.cc:2241] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.689836  1640 raft_consensus.cc:2270] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.676915  1230 raft_consensus.cc:383] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.690326  1230 raft_consensus.cc:738] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.691848   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet f845f32699044867af6f0f2b655cad8e
I20250628 07:52:26.692130  1640 ts_tablet_manager.cc:1905] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.694149  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet f845f32699044867af6f0f2b655cad8e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:26.695235   798 tablet_service.cc:1515] Processing DeleteTablet for tablet f845f32699044867af6f0f2b655cad8e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:26.695245   961 tablet_service.cc:1515] Processing DeleteTablet for tablet f845f32699044867af6f0f2b655cad8e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:26.696632  1229 consensus_queue.cc:260] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.690918  1230 consensus_queue.cc:260] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.702250  1229 ts_tablet_manager.cc:1428] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.039s	user 0.006s	sys 0.000s
I20250628 07:52:26.703109  1229 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:26.703442  1837 tablet.cc:1722] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:26.706091  1838 tablet.cc:1722] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:26.708995  1640 ts_tablet_manager.cc:1918] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:26.709836  1640 log.cc:1199] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:26.710583  1640 ts_tablet_manager.cc:1939] T cb62cf19aaf04444845358cc8ad611fb P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:26.713117  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 209) took 1233 ms. Trace:
I20250628 07:52:26.713451  1640 rpcz_store.cc:276] 0628 07:52:25.479406 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.479545 (+   139us) service_pool.cc:225] Handling call
0628 07:52:26.672925 (+1193380us) ts_tablet_manager.cc:1145] Deleting tablet cb62cf19aaf04444845358cc8ad611fb
0628 07:52:26.677145 (+  4220us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.700161 (+ 23016us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.708777 (+  8616us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.713041 (+  4264us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":11059,"tablet-delete.queue_time_us":1189133}
I20250628 07:52:26.714682   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet cb62cf19aaf04444845358cc8ad611fb (table default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]) successfully deleted
I20250628 07:52:26.715667  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 213) took 1191 ms. Trace:
I20250628 07:52:26.717271  1230 ts_tablet_manager.cc:1428] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.043s	user 0.006s	sys 0.000s
I20250628 07:52:26.724660  1244 ts_tablet_manager.cc:1918] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
W20250628 07:52:26.728631   920 consensus_peers.cc:489] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:26.728086  1128 raft_consensus.cc:1273] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:52:26.730286  1230 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:26.721415  1640 rpcz_store.cc:276] 0628 07:52:25.524696 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.524822 (+   126us) service_pool.cc:225] Handling call
0628 07:52:26.715377 (+1190555us) ts_tablet_manager.cc:1145] Deleting tablet 29a77cd9dc244d239a518e004c49e3c8
0628 07:52:26.715414 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.715623 (+   209us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1189992}
I20250628 07:52:26.734782  1779 consensus_queue.cc:1035] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:26.736207   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 29a77cd9dc244d239a518e004c49e3c8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29a77cd9dc244d239a518e004c49e3c8 already in progress: creating tablet
I20250628 07:52:26.737061  1640 tablet_replica.cc:331] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:26.737812  1640 raft_consensus.cc:2241] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.738694  1640 raft_consensus.cc:2270] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.739324  1807 ts_tablet_manager.cc:1918] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:26.739446  1244 log.cc:1199] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/2d77be1f37d2400f842c01019e4eb474
I20250628 07:52:26.740039  1807 log.cc:1199] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:26.740175  1244 ts_tablet_manager.cc:1939] T 2d77be1f37d2400f842c01019e4eb474 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:26.737919  1234 tablet_bootstrap.cc:654] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.744413  1640 ts_tablet_manager.cc:1905] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.745671  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 201) took 1788 ms. Trace:
I20250628 07:52:26.746021  1244 rpcz_store.cc:276] 0628 07:52:24.957488 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:24.961203 (+  3715us) service_pool.cc:225] Handling call
0628 07:52:26.573654 (+1612451us) ts_tablet_manager.cc:1145] Deleting tablet 2d77be1f37d2400f842c01019e4eb474
0628 07:52:26.573699 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.619835 (+ 46136us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.724460 (+104625us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.745603 (+ 21143us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1611960}
I20250628 07:52:26.747242  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 207) took 1089 ms. Trace:
I20250628 07:52:26.747220   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 2d77be1f37d2400f842c01019e4eb474 (table default.Table_55f07e6cb519419fa92ea6aeab0cad74 [id=15ceb9913eb84e44b89a2393cc7da9e3]) successfully deleted
I20250628 07:52:26.748414  1807 ts_tablet_manager.cc:1939] T 01feb8ef40dc4707bd6251fc2d87f46f P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:26.753706  1229 tablet_bootstrap.cc:654] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:26.747507  1244 rpcz_store.cc:276] 0628 07:52:25.658162 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.658277 (+   115us) service_pool.cc:225] Handling call
0628 07:52:26.746930 (+1088653us) ts_tablet_manager.cc:1145] Deleting tablet 29a77cd9dc244d239a518e004c49e3c8
0628 07:52:26.746967 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.747195 (+   228us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1088118}
I20250628 07:52:26.755785  1230 tablet_bootstrap.cc:654] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:26.755738   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 29a77cd9dc244d239a518e004c49e3c8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29a77cd9dc244d239a518e004c49e3c8 already in progress: creating tablet
I20250628 07:52:26.756453  1801 raft_consensus.cc:491] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.756829  1244 tablet_replica.cc:331] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:26.756870  1801 raft_consensus.cc:513] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.757503  1244 raft_consensus.cc:2241] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.757954  1244 raft_consensus.cc:2270] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.759438  1244 ts_tablet_manager.cc:1905] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.759713  1801 leader_election.cc:290] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:26.761399   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:26.764535  1062 leader_election.cc:343] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.766131   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:26.767355  1062 leader_election.cc:343] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.767695  1062 leader_election.cc:304] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:26.768685  1801 raft_consensus.cc:2747] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.782507   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 01feb8ef40dc4707bd6251fc2d87f46f (table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]) successfully deleted
I20250628 07:52:26.784221  1807 tablet_replica.cc:331] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:26.785866  1234 tablet_bootstrap.cc:492] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:26.786247  1234 ts_tablet_manager.cc:1397] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.132s	user 0.009s	sys 0.000s
I20250628 07:52:26.787222  1807 raft_consensus.cc:2241] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.793434  1807 raft_consensus.cc:2270] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.793247  1234 raft_consensus.cc:357] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.793844  1234 raft_consensus.cc:383] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.794097  1234 raft_consensus.cc:738] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.786253  1229 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:26.797408  1229 ts_tablet_manager.cc:1397] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.094s	user 0.014s	sys 0.000s
I20250628 07:52:26.794627  1234 consensus_queue.cc:260] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.799700  1229 raft_consensus.cc:357] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.805078  1837 tablet.cc:1722] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:26.800364  1229 raft_consensus.cc:383] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.807336  1229 raft_consensus.cc:738] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.808024  1229 consensus_queue.cc:260] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.809495  1838 tablet.cc:1722] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:26.810658  1229 ts_tablet_manager.cc:1428] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:26.799214  1807 ts_tablet_manager.cc:1905] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.814697   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:26.815338  1234 ts_tablet_manager.cc:1428] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.000s
I20250628 07:52:26.816174  1234 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:26.834327   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_78e2ac50470645f49e4cadc30ff61a59" } new_table_name: "default.Table_402049f057084c25a8b2a78d7fbdc2a0" modify_external_catalogs: true
I20250628 07:52:26.839748   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:26.847645  1229 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:26.853582   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_31674b85f9cd4f95ad5a60661a58f0b5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:26.854398   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9fc4e9c7ab7d413cba8e997e53c22844" } modify_external_catalogs: true
I20250628 07:52:26.854846   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9fc4e9c7ab7d413cba8e997e53c22844" } modify_external_catalogs: true
W20250628 07:52:26.855036   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_31674b85f9cd4f95ad5a60661a58f0b5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.856312   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_73dcf73d62f245089f066d28320dae9c" } new_table_name: "default.Table_39092c471b2443d183b7229a1d625b66" modify_external_catalogs: true
I20250628 07:52:26.858700   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7f61fed9e3094b7eb4f0fb5c23f5c721"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:26.860242   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7f61fed9e3094b7eb4f0fb5c23f5c721 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.864874  1801 raft_consensus.cc:491] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.865576  1801 raft_consensus.cc:513] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.868160  1801 leader_election.cc:290] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:26.870585   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:26.871663   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:26.872848  1062 leader_election.cc:343] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:26.873976  1062 leader_election.cc:343] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:26.874289  1062 leader_election.cc:304] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:26.875190  1801 raft_consensus.cc:2747] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.853504   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_528bf935d76946ddb897d5bf719bca19"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:26.878412   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_528bf935d76946ddb897d5bf719bca19 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:26.862164   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 46ff641001cd4b01a4c09c0d8b79ebf3 consensus state has no leader
I20250628 07:52:26.883877  1229 tablet_bootstrap.cc:654] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:26.892628   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8155990eebd84fcf9ebc357e186d1bf2 consensus state has no leader
I20250628 07:52:26.864873  1244 ts_tablet_manager.cc:1918] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:26.895337  1244 log.cc:1199] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/4e9d585d5f0c4469828561a7df4fb917
I20250628 07:52:26.896106  1244 ts_tablet_manager.cc:1939] T 4e9d585d5f0c4469828561a7df4fb917 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:26.896615  1840 raft_consensus.cc:491] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:26.896827  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 208) took 1135 ms. Trace:
I20250628 07:52:26.897162  1840 raft_consensus.cc:513] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.899035  1840 leader_election.cc:290] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:26.900312  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:26.900439   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8f7dde9b060f4d839724e5f718519fb1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:26.902185   758 leader_election.cc:304] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:26.903195  1840 raft_consensus.cc:2747] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:26.905131   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_31674b85f9cd4f95ad5a60661a58f0b5" } modify_external_catalogs: true
I20250628 07:52:26.905476  1230 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:26.905890  1230 ts_tablet_manager.cc:1397] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.176s	user 0.006s	sys 0.007s
I20250628 07:52:26.907119   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3eb1f00557094612af6b3582a0ce2679"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:26.908133  1230 raft_consensus.cc:357] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.905807   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_31674b85f9cd4f95ad5a60661a58f0b5" } modify_external_catalogs: true
I20250628 07:52:26.908721  1230 raft_consensus.cc:383] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.910053  1234 tablet_bootstrap.cc:654] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:26.913827   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3eb1f00557094612af6b3582a0ce2679 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:26.897164  1244 rpcz_store.cc:276] 0628 07:52:25.761364 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.763686 (+  2322us) service_pool.cc:225] Handling call
0628 07:52:26.756682 (+992996us) ts_tablet_manager.cc:1145] Deleting tablet 4e9d585d5f0c4469828561a7df4fb917
0628 07:52:26.756722 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.788539 (+ 31817us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.864660 (+ 76121us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.896766 (+ 32106us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":987345}
I20250628 07:52:26.916798   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 4e9d585d5f0c4469828561a7df4fb917 (table default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]) successfully deleted
I20250628 07:52:26.918627  1244 tablet_replica.cc:331] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:26.928839  1244 raft_consensus.cc:2241] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:26.929391  1244 raft_consensus.cc:2270] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:26.931859  1244 ts_tablet_manager.cc:1905] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:26.934433  1640 ts_tablet_manager.cc:1918] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:26.908977  1230 raft_consensus.cc:738] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.940029  1230 consensus_queue.cc:260] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.937785  1640 log.cc:1199] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/4e9d585d5f0c4469828561a7df4fb917
I20250628 07:52:26.942653  1640 ts_tablet_manager.cc:1939] T 4e9d585d5f0c4469828561a7df4fb917 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:26.967099  1234 tablet_bootstrap.cc:492] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:26.967484  1234 ts_tablet_manager.cc:1397] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.151s	user 0.006s	sys 0.002s
I20250628 07:52:26.954768  1230 ts_tablet_manager.cc:1428] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.049s	user 0.005s	sys 0.000s
I20250628 07:52:26.969861  1234 raft_consensus.cc:357] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.971895   544 master_service.cc:948] ReplaceTablet: received request to replace tablet 16a8bee148664e948529d32c0cf2c793 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:26.971895   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 9f479ac05fa44c089c060ebf2f2f6d07
I20250628 07:52:26.970415  1234 raft_consensus.cc:383] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:26.974253  1234 raft_consensus.cc:738] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:26.981628  1230 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:26.981863  1244 ts_tablet_manager.cc:1918] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:26.982411  1244 log.cc:1199] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/16a8bee148664e948529d32c0cf2c793
I20250628 07:52:26.974805  1234 consensus_queue.cc:260] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:26.985651  1244 ts_tablet_manager.cc:1939] T 16a8bee148664e948529d32c0cf2c793 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:26.967226  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 214) took 1396 ms. Trace:
I20250628 07:52:26.989181  1640 rpcz_store.cc:276] 0628 07:52:25.570532 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.570665 (+   133us) service_pool.cc:225] Handling call
0628 07:52:26.736899 (+1166234us) ts_tablet_manager.cc:1145] Deleting tablet 4e9d585d5f0c4469828561a7df4fb917
0628 07:52:26.736954 (+    55us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.820588 (+ 83634us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.934217 (+113629us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.967162 (+ 32945us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1165567}
I20250628 07:52:26.990398   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 4e9d585d5f0c4469828561a7df4fb917 (table default.Table_c8d2155d54dc47c0a639ace22ba757eb [id=44a9edd2805b47e1b19b5709d57770bd]) successfully deleted
I20250628 07:52:26.994318  1234 ts_tablet_manager.cc:1428] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.027s	user 0.005s	sys 0.001s
I20250628 07:52:26.994230  1102 tablet_service.cc:1468] Processing CreateTablet for tablet a8518c82c188484c8554d3115637f5be (DEFAULT_TABLE table=default.Table_32928f5c3c044deaa68ad9b7971a3aaf [id=65bbed71d6394e34a7a2f3f3f5fa069d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:26.995251  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8518c82c188484c8554d3115637f5be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:26.995849  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 209) took 1232 ms. Trace:
I20250628 07:52:27.006304   798 tablet_service.cc:1468] Processing CreateTablet for tablet a8518c82c188484c8554d3115637f5be (DEFAULT_TABLE table=default.Table_32928f5c3c044deaa68ad9b7971a3aaf [id=65bbed71d6394e34a7a2f3f3f5fa069d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.008445  1234 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.009001   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8518c82c188484c8554d3115637f5be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.011268  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 221) took 1039 ms. Trace:
I20250628 07:52:26.997720   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:27.011574  1640 rpcz_store.cc:276] 0628 07:52:25.972116 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.972228 (+   112us) service_pool.cc:225] Handling call
0628 07:52:27.010934 (+1038706us) ts_tablet_manager.cc:1145] Deleting tablet b2d49e5a06384758bce5e7328b6a10f8
0628 07:52:27.010977 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.011214 (+   237us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1038145}
W20250628 07:52:27.012635   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b2d49e5a06384758bce5e7328b6a10f8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b2d49e5a06384758bce5e7328b6a10f8 already in progress: creating tablet
I20250628 07:52:27.014925  1234 tablet_bootstrap.cc:654] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.020030   801 tablet_service.cc:1468] Processing CreateTablet for tablet a353cc9eedf34cc7aa4cfbcc7198247b (DEFAULT_TABLE table=default.Table_09dfad9563454a0e9624d7e15961a374 [id=dce9747fa94f4f86bdafae93dbf53339]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.020942   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a353cc9eedf34cc7aa4cfbcc7198247b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.022712  1230 tablet_bootstrap.cc:654] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.025161  1244 rpcz_store.cc:276] 0628 07:52:25.763543 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.773831 (+ 10288us) service_pool.cc:225] Handling call
0628 07:52:26.918469 (+1144638us) ts_tablet_manager.cc:1145] Deleting tablet 16a8bee148664e948529d32c0cf2c793
0628 07:52:26.918515 (+    46us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.962487 (+ 43972us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.981659 (+ 19172us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:26.995768 (+ 14109us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":7991,"tablet-delete.queue_time_us":1144060}
I20250628 07:52:27.026279  1244 tablet_replica.cc:331] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:27.027927  1244 raft_consensus.cc:2241] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.028200   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 16a8bee148664e948529d32c0cf2c793 (table default.Table_0adee4e1249840be952b0a99dba4c161 [id=f59d42b137cf4536a002585b728017f6]) successfully deleted
I20250628 07:52:27.028349  1244 raft_consensus.cc:2270] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.029948  1244 ts_tablet_manager.cc:1905] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.034101  1101 tablet_service.cc:1468] Processing CreateTablet for tablet a353cc9eedf34cc7aa4cfbcc7198247b (DEFAULT_TABLE table=default.Table_09dfad9563454a0e9624d7e15961a374 [id=dce9747fa94f4f86bdafae93dbf53339]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.035816   799 tablet_service.cc:1468] Processing CreateTablet for tablet f8bce65d32e042ab89ebe168d9657466 (DEFAULT_TABLE table=default.Table_b66cef7061354731bda56976a11f758d [id=380a32248ae74b7697746c5853248303]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.037266   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8bce65d32e042ab89ebe168d9657466. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.039335  1108 tablet_service.cc:1468] Processing CreateTablet for tablet f8bce65d32e042ab89ebe168d9657466 (DEFAULT_TABLE table=default.Table_b66cef7061354731bda56976a11f758d [id=380a32248ae74b7697746c5853248303]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.040740   804 tablet_service.cc:1468] Processing CreateTablet for tablet e52e15aefe194a61baa7e74616d9bfd3 (DEFAULT_TABLE table=default.Table_5d099833475e4b44b3fda7923e027a37 [id=d49f6e667b6e409c9f8f225158c16236]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.041710   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e52e15aefe194a61baa7e74616d9bfd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.047528  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8bce65d32e042ab89ebe168d9657466. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.049695  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a353cc9eedf34cc7aa4cfbcc7198247b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.061398  1102 tablet_service.cc:1468] Processing CreateTablet for tablet e52e15aefe194a61baa7e74616d9bfd3 (DEFAULT_TABLE table=default.Table_5d099833475e4b44b3fda7923e027a37 [id=d49f6e667b6e409c9f8f225158c16236]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.062575  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e52e15aefe194a61baa7e74616d9bfd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.066661   798 tablet_service.cc:1515] Processing DeleteTablet for tablet e4e02676d24a4345908dbbc42fc9c1ab with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:27.071476  1234 tablet_bootstrap.cc:492] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.071902  1234 ts_tablet_manager.cc:1397] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.064s	user 0.008s	sys 0.000s
I20250628 07:52:27.072099   798 tablet_service.cc:1468] Processing CreateTablet for tablet 5f77a64b8e054f45acbad44d02af0e82 (DEFAULT_TABLE table=default.Table_9775925de3b34654b11d221bc66cb7fd [id=1f4b61bbe4284a718e2adc4e1a1fcc59]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.073895  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 5f77a64b8e054f45acbad44d02af0e82 (DEFAULT_TABLE table=default.Table_9775925de3b34654b11d221bc66cb7fd [id=1f4b61bbe4284a718e2adc4e1a1fcc59]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.074103  1234 raft_consensus.cc:357] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.074780  1234 raft_consensus.cc:383] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.074923  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f77a64b8e054f45acbad44d02af0e82. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.075037  1234 raft_consensus.cc:738] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.081280  1640 tablet_replica.cc:331] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:27.082070  1640 raft_consensus.cc:2241] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.082496  1640 raft_consensus.cc:2270] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.084512  1640 ts_tablet_manager.cc:1905] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.085321  1234 consensus_queue.cc:260] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.087208  1234 ts_tablet_manager.cc:1428] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
I20250628 07:52:27.094094   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ade87cf74e6b498690118e3541e41383"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:27.095952  1234 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.095359   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1f0b83a168f54608ae33309446b7b2aa" } new_table_name: "default.Table_d22b2c7f99204fd68d8a98c993ea9f40" modify_external_catalogs: true
W20250628 07:52:27.101097   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ade87cf74e6b498690118e3541e41383 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.103533   542 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6ce1e375ffee4e2db52c58c171ba4d08" } new_table_name: "default.Table_10140fc26c314afab000f5f4626f9110" modify_external_catalogs: true
I20250628 07:52:27.106848  1244 ts_tablet_manager.cc:1918] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.107216  1244 log.cc:1199] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/1d2c0b7e3d9e448ebb2fe3530da7bbba
I20250628 07:52:27.107918  1244 ts_tablet_manager.cc:1939] T 1d2c0b7e3d9e448ebb2fe3530da7bbba P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
W20250628 07:52:27.108829   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (table default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 46ff641001cd4b01a4c09c0d8b79ebf3 consensus state has no leader
I20250628 07:52:27.109414   538 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:27.111292   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:27.112012   799 tablet_service.cc:1468] Processing CreateTablet for tablet fb3d9cc430354e54b82ed7844fd2181c (DEFAULT_TABLE table=default.Table_18695aa15f7843b9a5388e863015aae0 [id=5be853f3c9e74f0a8cebb878f97e4f62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.115032   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f77a64b8e054f45acbad44d02af0e82. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.118573  1109 tablet_service.cc:1468] Processing CreateTablet for tablet fb3d9cc430354e54b82ed7844fd2181c (DEFAULT_TABLE table=default.Table_18695aa15f7843b9a5388e863015aae0 [id=5be853f3c9e74f0a8cebb878f97e4f62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.119486  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3d9cc430354e54b82ed7844fd2181c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.122186   800 tablet_service.cc:1468] Processing CreateTablet for tablet 3519e872e8504a4fb962814e3438bc01 (DEFAULT_TABLE table=default.Table_49dd1c44ec2d43b9bdfe6c8169f90406 [id=1478e98be7a24e5595821aea5ec4ee7c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.122901  1234 tablet_bootstrap.cc:654] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.129160   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:27.130331   543 catalog_manager.cc:6652] ReplaceTablet: tablet 2ae890710f7f44cf899dd19e96fa9181 deleted and replaced by tablet 66353977776545d4a712f58db2aff108
I20250628 07:52:27.132519   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3d9cc430354e54b82ed7844fd2181c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.137782   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:27.140630  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:27.145773   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3519e872e8504a4fb962814e3438bc01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.145551   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e10aee64839f49a18fc40406bb808709"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:27.145491   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d65bc763056e4ce181baccde39c92ab9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:27.146924   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e10aee64839f49a18fc40406bb808709 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:27.147070   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d65bc763056e4ce181baccde39c92ab9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.147913  1230 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:27.148284  1230 ts_tablet_manager.cc:1397] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.167s	user 0.005s	sys 0.004s
I20250628 07:52:27.151417   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]). Expected version 1 got 0
I20250628 07:52:27.158382  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 210) took 1369 ms. Trace:
I20250628 07:52:27.158763  1244 rpcz_store.cc:276] 0628 07:52:25.788486 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:25.814384 (+ 25898us) service_pool.cc:225] Handling call
0628 07:52:27.026148 (+1211764us) ts_tablet_manager.cc:1145] Deleting tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba
0628 07:52:27.026186 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.058270 (+ 32084us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.106657 (+ 48387us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.158325 (+ 51668us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":947,"tablet-delete.queue_time_us":1211105}
I20250628 07:52:27.159994   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 1d2c0b7e3d9e448ebb2fe3530da7bbba (table default.Table_c6657dd54c2f4decb3eda54e473d4c10 [id=b63408194ba04deb9119657ffbca781f]) successfully deleted
I20250628 07:52:27.160437  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 217) took 1020 ms. Trace:
I20250628 07:52:27.160760  1244 rpcz_store.cc:276] 0628 07:52:26.139588 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.139708 (+   120us) service_pool.cc:225] Handling call
0628 07:52:27.160060 (+1020352us) ts_tablet_manager.cc:1145] Deleting tablet b2d49e5a06384758bce5e7328b6a10f8
0628 07:52:27.160097 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.160373 (+   276us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1003966}
W20250628 07:52:27.161831   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b2d49e5a06384758bce5e7328b6a10f8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b2d49e5a06384758bce5e7328b6a10f8 already in progress: creating tablet
I20250628 07:52:27.162035  1244 tablet_replica.cc:331] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:27.162680  1234 tablet_bootstrap.cc:492] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.162636  1230 raft_consensus.cc:357] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.163064  1234 ts_tablet_manager.cc:1397] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.067s	user 0.007s	sys 0.002s
I20250628 07:52:27.163251  1230 raft_consensus.cc:383] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.163491  1230 raft_consensus.cc:738] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.164038  1230 consensus_queue.cc:260] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.164934   797 tablet_service.cc:1468] Processing CreateTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd (DEFAULT_TABLE table=default.Table_e26652b4c0454211b542434dc5992cd9 [id=ec582a4acf604066baa68b5762ff95bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.165063  1234 raft_consensus.cc:357] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.165648  1234 raft_consensus.cc:383] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.165866  1234 raft_consensus.cc:738] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.165932   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fd67d7d9ba9446c19f1711fb5bd73ebd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.166334  1234 consensus_queue.cc:260] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.162822  1244 raft_consensus.cc:2241] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:27.178591  1244 pending_rounds.cc:70] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Trying to abort 1 pending ops.
I20250628 07:52:27.178823  1244 pending_rounds.cc:77] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493912433541120 op_type: NO_OP noop_request { }
I20250628 07:52:27.179138  1244 raft_consensus.cc:2887] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:27.179438  1244 raft_consensus.cc:2270] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.181926   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_06cb6f2bb04a4517ad00c1c925fe4ef9" } modify_external_catalogs: true
I20250628 07:52:27.182385   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_06cb6f2bb04a4517ad00c1c925fe4ef9" } modify_external_catalogs: true
I20250628 07:52:27.182302  1107 tablet_service.cc:1468] Processing CreateTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd (DEFAULT_TABLE table=default.Table_e26652b4c0454211b542434dc5992cd9 [id=ec582a4acf604066baa68b5762ff95bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.182736   799 tablet_service.cc:1468] Processing CreateTablet for tablet 613eb4aafd7b4a93b0f2a9699a942f34 (DEFAULT_TABLE table=default.Table_9e1de6bb51104538894960ac45e8b097 [id=d537b3af320d4a858ba125b23bfe0fa2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.183210  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fd67d7d9ba9446c19f1711fb5bd73ebd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.185612  1244 ts_tablet_manager.cc:1905] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.190876   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:27.193825   804 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:27.198201  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:27.203132  1230 ts_tablet_manager.cc:1428] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.055s	user 0.003s	sys 0.002s
I20250628 07:52:27.203948   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 613eb4aafd7b4a93b0f2a9699a942f34. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.210853  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 613eb4aafd7b4a93b0f2a9699a942f34 (DEFAULT_TABLE table=default.Table_9e1de6bb51104538894960ac45e8b097 [id=d537b3af320d4a858ba125b23bfe0fa2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.211730  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 613eb4aafd7b4a93b0f2a9699a942f34. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.218417  1234 ts_tablet_manager.cc:1428] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.055s	user 0.001s	sys 0.005s
I20250628 07:52:27.219306  1234 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.219642  1230 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:27.224810   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5475a89692eb450d82cde521ec8c9a73 consensus state has no leader
I20250628 07:52:27.251798   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111), 
I20250628 07:52:27.252218   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:26.984140  1229 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:27.256052  1229 ts_tablet_manager.cc:1397] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.408s	user 0.014s	sys 0.005s
I20250628 07:52:27.262919  1234 tablet_bootstrap.cc:654] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.260179  1229 raft_consensus.cc:357] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.264521  1229 raft_consensus.cc:383] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.264828  1229 raft_consensus.cc:738] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.265527  1229 consensus_queue.cc:260] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.266602   801 tablet_service.cc:1468] Processing CreateTablet for tablet d9e0ea5e300347c7adc8e30b9070976b (DEFAULT_TABLE table=default.Table_a374b2114f7447f392f342b445cd482d [id=6c3c849b0c8b437eac1952cc7457d1be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.267915   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d9e0ea5e300347c7adc8e30b9070976b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.268982  1102 tablet_service.cc:1468] Processing CreateTablet for tablet d9e0ea5e300347c7adc8e30b9070976b (DEFAULT_TABLE table=default.Table_a374b2114f7447f392f342b445cd482d [id=6c3c849b0c8b437eac1952cc7457d1be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.270294  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d9e0ea5e300347c7adc8e30b9070976b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.280791   800 tablet_service.cc:1468] Processing CreateTablet for tablet 181c57eeb50745beb09c36a38b75d9be (DEFAULT_TABLE table=default.Table_c81c2135bc5a4417b706d1c042207cc0 [id=0397ad0e0e114f359480497eac5e7712]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.281862   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 181c57eeb50745beb09c36a38b75d9be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.298658   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_df7fc4695669432f9b2bcc300360c70b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:27.299374  1807 ts_tablet_manager.cc:1918] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.300197  1807 log.cc:1199] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/e4e02676d24a4345908dbbc42fc9c1ab
W20250628 07:52:27.300235   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_df7fc4695669432f9b2bcc300360c70b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.301162  1807 ts_tablet_manager.cc:1939] T e4e02676d24a4345908dbbc42fc9c1ab P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:27.301251   546 master_service.cc:948] ReplaceTablet: received request to replace tablet 46ff641001cd4b01a4c09c0d8b79ebf3 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:27.310424  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 3519e872e8504a4fb962814e3438bc01 (DEFAULT_TABLE table=default.Table_49dd1c44ec2d43b9bdfe6c8169f90406 [id=1478e98be7a24e5595821aea5ec4ee7c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.311779  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3519e872e8504a4fb962814e3438bc01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.319926  1234 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.319980   818 raft_consensus.cc:3058] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.320320  1234 ts_tablet_manager.cc:1397] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.101s	user 0.008s	sys 0.000s
I20250628 07:52:27.323915  1234 raft_consensus.cc:357] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.324466  1234 raft_consensus.cc:383] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.324705  1234 raft_consensus.cc:738] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:27.326474   542 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet d87dd96d00964e12acd6e380a62c94d8 consensus state has no leader
I20250628 07:52:27.339594  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 222) took 1308 ms. Trace:
I20250628 07:52:27.340072  1807 rpcz_store.cc:276] 0628 07:52:26.030659 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.030773 (+   114us) service_pool.cc:225] Handling call
0628 07:52:26.784031 (+753258us) ts_tablet_manager.cc:1145] Deleting tablet e4e02676d24a4345908dbbc42fc9c1ab
0628 07:52:26.784093 (+    62us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:26.919688 (+135595us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.298977 (+379289us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.339459 (+ 40482us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":752725}
I20250628 07:52:27.350071  1234 consensus_queue.cc:260] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.352283  1234 ts_tablet_manager.cc:1428] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.005s	sys 0.001s
I20250628 07:52:27.365586  1229 ts_tablet_manager.cc:1428] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.109s	user 0.006s	sys 0.006s
I20250628 07:52:27.366636  1229 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:27.370512  1234 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.371608  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 181c57eeb50745beb09c36a38b75d9be (DEFAULT_TABLE table=default.Table_c81c2135bc5a4417b706d1c042207cc0 [id=0397ad0e0e114f359480497eac5e7712]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.372691  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 181c57eeb50745beb09c36a38b75d9be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.375115  1106 tablet_service.cc:1468] Processing CreateTablet for tablet f9a553c69c254fdcaf413b53018bbfb2 (DEFAULT_TABLE table=default.Table_b0c733c64d5449ea86e9103a47c2f578 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.376047  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9a553c69c254fdcaf413b53018bbfb2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.386077  1234 tablet_bootstrap.cc:654] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.397819   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7f61fed9e3094b7eb4f0fb5c23f5c721" } modify_external_catalogs: true
I20250628 07:52:27.398319   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7f61fed9e3094b7eb4f0fb5c23f5c721" } modify_external_catalogs: true
I20250628 07:52:27.400487   824 raft_consensus.cc:3058] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.410048   538 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f] alter complete (version 1)
I20250628 07:52:27.411700  1234 tablet_bootstrap.cc:492] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.412108  1234 ts_tablet_manager.cc:1397] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.059s	user 0.008s	sys 0.000s
I20250628 07:52:27.422390   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet e4e02676d24a4345908dbbc42fc9c1ab (table default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]) successfully deleted
I20250628 07:52:27.424353   961 tablet_service.cc:1468] Processing CreateTablet for tablet a8518c82c188484c8554d3115637f5be (DEFAULT_TABLE table=default.Table_32928f5c3c044deaa68ad9b7971a3aaf [id=65bbed71d6394e34a7a2f3f3f5fa069d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.425750   960 tablet_service.cc:1468] Processing CreateTablet for tablet a353cc9eedf34cc7aa4cfbcc7198247b (DEFAULT_TABLE table=default.Table_09dfad9563454a0e9624d7e15961a374 [id=dce9747fa94f4f86bdafae93dbf53339]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.427457   963 tablet_service.cc:1468] Processing CreateTablet for tablet f8bce65d32e042ab89ebe168d9657466 (DEFAULT_TABLE table=default.Table_b66cef7061354731bda56976a11f758d [id=380a32248ae74b7697746c5853248303]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.428117   965 tablet_service.cc:1468] Processing CreateTablet for tablet 5f77a64b8e054f45acbad44d02af0e82 (DEFAULT_TABLE table=default.Table_9775925de3b34654b11d221bc66cb7fd [id=1f4b61bbe4284a718e2adc4e1a1fcc59]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.428813   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8518c82c188484c8554d3115637f5be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.428867   959 tablet_service.cc:1468] Processing CreateTablet for tablet e52e15aefe194a61baa7e74616d9bfd3 (DEFAULT_TABLE table=default.Table_5d099833475e4b44b3fda7923e027a37 [id=d49f6e667b6e409c9f8f225158c16236]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.429554   966 tablet_service.cc:1468] Processing CreateTablet for tablet fb3d9cc430354e54b82ed7844fd2181c (DEFAULT_TABLE table=default.Table_18695aa15f7843b9a5388e863015aae0 [id=5be853f3c9e74f0a8cebb878f97e4f62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.429710  1234 raft_consensus.cc:357] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.430356  1234 raft_consensus.cc:383] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.430634  1234 raft_consensus.cc:738] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.430859   962 tablet_service.cc:1468] Processing CreateTablet for tablet 3519e872e8504a4fb962814e3438bc01 (DEFAULT_TABLE table=default.Table_49dd1c44ec2d43b9bdfe6c8169f90406 [id=1478e98be7a24e5595821aea5ec4ee7c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.431382  1234 consensus_queue.cc:260] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.435478  1640 ts_tablet_manager.cc:1918] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.436684   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a353cc9eedf34cc7aa4cfbcc7198247b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.439136  1807 tablet_replica.cc:331] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:27.440166  1807 raft_consensus.cc:2241] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.440732  1807 raft_consensus.cc:2270] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.442134   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f77a64b8e054f45acbad44d02af0e82. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.442668   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3d9cc430354e54b82ed7844fd2181c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.443246   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8bce65d32e042ab89ebe168d9657466. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.444471   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3519e872e8504a4fb962814e3438bc01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.444983   959 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e52e15aefe194a61baa7e74616d9bfd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.461568   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:27.462556   958 tablet_service.cc:1468] Processing CreateTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd (DEFAULT_TABLE table=default.Table_e26652b4c0454211b542434dc5992cd9 [id=ec582a4acf604066baa68b5762ff95bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.463743   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fd67d7d9ba9446c19f1711fb5bd73ebd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.468151   797 tablet_service.cc:1468] Processing CreateTablet for tablet f9a553c69c254fdcaf413b53018bbfb2 (DEFAULT_TABLE table=default.Table_b0c733c64d5449ea86e9103a47c2f578 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.469306   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9a553c69c254fdcaf413b53018bbfb2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.470505   964 tablet_service.cc:1468] Processing CreateTablet for tablet 613eb4aafd7b4a93b0f2a9699a942f34 (DEFAULT_TABLE table=default.Table_9e1de6bb51104538894960ac45e8b097 [id=d537b3af320d4a858ba125b23bfe0fa2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.471092   957 tablet_service.cc:1468] Processing CreateTablet for tablet d9e0ea5e300347c7adc8e30b9070976b (DEFAULT_TABLE table=default.Table_a374b2114f7447f392f342b445cd482d [id=6c3c849b0c8b437eac1952cc7457d1be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.471499   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 613eb4aafd7b4a93b0f2a9699a942f34. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.472105   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d9e0ea5e300347c7adc8e30b9070976b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.474061   956 tablet_service.cc:1468] Processing CreateTablet for tablet 181c57eeb50745beb09c36a38b75d9be (DEFAULT_TABLE table=default.Table_c81c2135bc5a4417b706d1c042207cc0 [id=0397ad0e0e114f359480497eac5e7712]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.475070   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 181c57eeb50745beb09c36a38b75d9be. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.479130   955 tablet_service.cc:1468] Processing CreateTablet for tablet f9a553c69c254fdcaf413b53018bbfb2 (DEFAULT_TABLE table=default.Table_b0c733c64d5449ea86e9103a47c2f578 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:27.480152   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9a553c69c254fdcaf413b53018bbfb2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:27.487013  1234 ts_tablet_manager.cc:1428] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.075s	user 0.007s	sys 0.000s
I20250628 07:52:27.487964  1234 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.491042  1840 raft_consensus.cc:491] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.491436  1840 raft_consensus.cc:513] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.493858   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_14e4b6f1c4f344a280e1901e7f772fbd" } new_table_name: "default.Table_52441cbec305498ebed774e69004a383" modify_external_catalogs: true
I20250628 07:52:27.494071  1840 leader_election.cc:290] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.494431  1840 raft_consensus.cc:491] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.494769  1840 raft_consensus.cc:513] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:27.498240   758 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:27.503139   542 catalog_manager.cc:5582] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:27.504016  1840 leader_election.cc:290] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.505801   542 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_5e720c1e9255472196788707be8302ec [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 1 got 0
I20250628 07:52:27.508548  1234 tablet_bootstrap.cc:654] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.515753  1840 raft_consensus.cc:491] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.516269  1840 raft_consensus.cc:513] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.519117  1840 leader_election.cc:290] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:27.520961   758 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:27.529338   966 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:27.535895  1876 tablet.cc:1722] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:27.553702   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_5e720c1e9255472196788707be8302ec" } new_table_name: "default.Table_a6c8170745a346f9bc81b8e023e95b7c" modify_external_catalogs: true
I20250628 07:52:27.558521   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:27.560098   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f29c8ffb7df410295278eee9c578b03" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:27.555771  1878 tablet.cc:1722] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
W20250628 07:52:27.565596   757 leader_election.cc:343] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:27.566080   757 leader_election.cc:304] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:27.568710  1234 tablet_bootstrap.cc:492] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.569247  1234 ts_tablet_manager.cc:1397] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.081s	user 0.008s	sys 0.003s
I20250628 07:52:27.571467  1234 raft_consensus.cc:357] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.503093  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "14fedcf110e64c33978004743c5096ed" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:27.572892  1234 raft_consensus.cc:383] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.573174  1234 raft_consensus.cc:738] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.573307  1125 raft_consensus.cc:2391] T 14fedcf110e64c33978004743c5096ed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:27.508472  1801 raft_consensus.cc:491] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.573827  1234 consensus_queue.cc:260] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.574134  1801 raft_consensus.cc:513] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.574458   758 leader_election.cc:304] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:27.576007  1801 leader_election.cc:290] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:27.576529  1640 log.cc:1199] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:27.577471  1640 ts_tablet_manager.cc:1939] T f42b22f7096e46cf9cc09d1caecc9efb P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:27.578317  1879 raft_consensus.cc:2747] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:27.578680   757 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:27.581359   757 leader_election.cc:304] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:27.588521  1879 raft_consensus.cc:3058] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.594357   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:27.596581  1062 leader_election.cc:343] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:27.604472   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4eb55153d17a4680b1de4c5ecba431d2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:27.606243   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4eb55153d17a4680b1de4c5ecba431d2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.607648  1640 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 224) took 1230 ms. Trace:
I20250628 07:52:27.607966  1640 rpcz_store.cc:276] 0628 07:52:26.376950 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.384711 (+  7761us) service_pool.cc:225] Handling call
0628 07:52:27.016708 (+631997us) ts_tablet_manager.cc:1145] Deleting tablet f42b22f7096e46cf9cc09d1caecc9efb
0628 07:52:27.081160 (+ 64452us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.170674 (+ 89514us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.435248 (+264574us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.607586 (+172338us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":631437}
I20250628 07:52:27.609035   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet f42b22f7096e46cf9cc09d1caecc9efb (table default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]) successfully deleted
W20250628 07:52:27.610915   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet f845f32699044867af6f0f2b655cad8e because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f845f32699044867af6f0f2b655cad8e already in progress: creating tablet
I20250628 07:52:27.615298  1244 ts_tablet_manager.cc:1918] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:27.615720  1244 log.cc:1199] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:27.616389  1244 ts_tablet_manager.cc:1939] T fab31b073e2e49318b4118851f81b6d2 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:27.626578  1230 tablet_bootstrap.cc:654] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:27.627048   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 2ae890710f7f44cf899dd19e96fa9181 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2ae890710f7f44cf899dd19e96fa9181 already in progress: creating tablet
W20250628 07:52:27.630105   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet cc0bf0edebb140209531ccf917fd1ed8 already in progress: creating tablet
I20250628 07:52:27.638934   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet fab31b073e2e49318b4118851f81b6d2 (table default.Table_1935e83524ad453c8175710332a3c459 [id=491d07dd82d54928b39605ba6d28af42]) successfully deleted
I20250628 07:52:27.639737  1244 tablet_replica.cc:331] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:27.640564  1244 raft_consensus.cc:2241] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.641103  1244 raft_consensus.cc:2270] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.648097  1879 raft_consensus.cc:2747] T 14fedcf110e64c33978004743c5096ed P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:27.650483  1244 ts_tablet_manager.cc:1905] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.658087   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:27.660009  1807 ts_tablet_manager.cc:1905] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.663283  1230 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:27.663761  1230 ts_tablet_manager.cc:1397] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.444s	user 0.013s	sys 0.000s
I20250628 07:52:27.665999  1230 raft_consensus.cc:357] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:27.671343   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 96972.7us
I20250628 07:52:27.665290  1875 raft_consensus.cc:2747] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:27.682338  1244 ts_tablet_manager.cc:1918] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.682626  1244 log.cc:1199] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:27.683116  1244 ts_tablet_manager.cc:1939] T f42b22f7096e46cf9cc09d1caecc9efb P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:27.683741  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 221) took 1052 ms. Trace:
I20250628 07:52:27.683993  1244 rpcz_store.cc:276] 0628 07:52:26.631815 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.631937 (+   122us) service_pool.cc:225] Handling call
0628 07:52:27.639584 (+1007647us) ts_tablet_manager.cc:1145] Deleting tablet f42b22f7096e46cf9cc09d1caecc9efb
0628 07:52:27.639626 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.674334 (+ 34708us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.682171 (+  7837us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.683698 (+  1527us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1007103}
I20250628 07:52:27.685531   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet f42b22f7096e46cf9cc09d1caecc9efb (table default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]) successfully deleted
I20250628 07:52:27.696787  1234 ts_tablet_manager.cc:1428] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.127s	user 0.012s	sys 0.015s
I20250628 07:52:27.697618  1234 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:27.701833   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 123183us
I20250628 07:52:27.705433  1244 tablet_replica.cc:331] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:27.706369  1244 raft_consensus.cc:2241] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.706748  1244 raft_consensus.cc:2270] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.708283  1244 ts_tablet_manager.cc:1905] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.709292  1234 tablet_bootstrap.cc:654] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.726708   540 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 94cb4319bf9c4c98b361dbee522874f5
I20250628 07:52:27.727571   540 master_service.cc:948] ReplaceTablet: received request to replace tablet 2ae890710f7f44cf899dd19e96fa9181 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:27.756439  1234 tablet_bootstrap.cc:492] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.756800  1234 ts_tablet_manager.cc:1397] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.059s	user 0.009s	sys 0.010s
I20250628 07:52:27.758746  1234 raft_consensus.cc:357] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.759320  1234 raft_consensus.cc:383] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.759544  1234 raft_consensus.cc:738] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.661291  1229 tablet_bootstrap.cc:654] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.681509  1866 raft_consensus.cc:491] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.760114  1234 consensus_queue.cc:260] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.760043  1866 raft_consensus.cc:513] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.762194  1866 leader_election.cc:290] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.762552  1234 ts_tablet_manager.cc:1428] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.001s
I20250628 07:52:27.694095  1864 consensus_queue.cc:1035] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:27.761219  1244 ts_tablet_manager.cc:1918] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.764982  1244 log.cc:1199] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/01feb8ef40dc4707bd6251fc2d87f46f
I20250628 07:52:27.716732  1863 raft_consensus.cc:491] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.769596  1863 raft_consensus.cc:513] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.771646  1863 leader_election.cc:290] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.773689  1234 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.776608  1244 ts_tablet_manager.cc:1939] T 01feb8ef40dc4707bd6251fc2d87f46f P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:27.778288  1234 tablet_bootstrap.cc:654] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:27.779735   920 leader_election.cc:343] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:27.780630  1865 consensus_queue.cc:1035] T 14fedcf110e64c33978004743c5096ed P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:27.785419  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 222) took 1119 ms. Trace:
I20250628 07:52:27.785763  1244 rpcz_store.cc:276] 0628 07:52:26.666281 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.666399 (+   118us) service_pool.cc:225] Handling call
0628 07:52:27.705298 (+1038899us) ts_tablet_manager.cc:1145] Deleting tablet 01feb8ef40dc4707bd6251fc2d87f46f
0628 07:52:27.705340 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.735053 (+ 29713us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.761018 (+ 25965us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:27.785066 (+ 24048us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1038414}
I20250628 07:52:27.785336   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:27.786725   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 01feb8ef40dc4707bd6251fc2d87f46f (table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]) successfully deleted
I20250628 07:52:27.786832   818 raft_consensus.cc:2466] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:27.788446   920 leader_election.cc:304] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:27.789388  1863 raft_consensus.cc:2802] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:27.789765  1863 raft_consensus.cc:491] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:27.790146  1863 raft_consensus.cc:3058] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.793584  1244 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 223) took 1098 ms. Trace:
I20250628 07:52:27.793927  1244 rpcz_store.cc:276] 0628 07:52:26.694956 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.695106 (+   150us) service_pool.cc:225] Handling call
0628 07:52:27.793291 (+1098185us) ts_tablet_manager.cc:1145] Deleting tablet f845f32699044867af6f0f2b655cad8e
0628 07:52:27.793328 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:27.793526 (+   198us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1097700}
W20250628 07:52:27.795002   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet f845f32699044867af6f0f2b655cad8e because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f845f32699044867af6f0f2b655cad8e already in progress: creating tablet
W20250628 07:52:27.796319   921 leader_election.cc:343] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:27.796778   921 leader_election.cc:304] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:27.797062  1244 tablet_replica.cc:331] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:27.797529  1865 raft_consensus.cc:2747] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:27.797859  1244 raft_consensus.cc:2241] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:27.798528  1234 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.806224  1234 ts_tablet_manager.cc:1397] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.033s	user 0.006s	sys 0.005s
I20250628 07:52:27.806246  1244 raft_consensus.cc:2270] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:27.808248  1234 raft_consensus.cc:357] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.808846  1234 raft_consensus.cc:383] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.809105  1234 raft_consensus.cc:738] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.810098  1229 tablet_bootstrap.cc:492] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:27.809830  1234 consensus_queue.cc:260] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.810487  1229 ts_tablet_manager.cc:1397] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.444s	user 0.006s	sys 0.004s
I20250628 07:52:27.810949  1244 ts_tablet_manager.cc:1905] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:27.812300  1234 ts_tablet_manager.cc:1428] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20250628 07:52:27.813118  1234 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.815107   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d8ead90f917d46b9beef6badf9823be1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:27.817026   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d8ead90f917d46b9beef6badf9823be1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.812665  1229 raft_consensus.cc:357] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.818672  1229 raft_consensus.cc:383] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.818934  1229 raft_consensus.cc:738] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.817751  1234 tablet_bootstrap.cc:654] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.821362  1229 consensus_queue.cc:260] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.836804  1234 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.837198  1234 ts_tablet_manager.cc:1397] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.024s	user 0.010s	sys 0.000s
I20250628 07:52:27.839581  1234 raft_consensus.cc:357] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.840226  1234 raft_consensus.cc:383] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.840476  1234 raft_consensus.cc:738] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.841207  1234 consensus_queue.cc:260] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:27.685794  1062 leader_election.cc:343] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:27.843976  1062 leader_election.cc:304] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:27.844894  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 159120us
I20250628 07:52:27.769773  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:27.845307  1234 ts_tablet_manager.cc:1428] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.000s
I20250628 07:52:27.845680  1125 raft_consensus.cc:2466] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:27.846464  1801 raft_consensus.cc:2747] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:27.847143  1230 raft_consensus.cc:383] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:27.848500  1230 raft_consensus.cc:738] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.849310  1230 consensus_queue.cc:260] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.852854  1229 ts_tablet_manager.cc:1428] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.042s	user 0.006s	sys 0.000s
I20250628 07:52:27.856271  1863 raft_consensus.cc:513] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.857321  1234 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.864645   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:27.865250   818 raft_consensus.cc:3058] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.865502  1234 tablet_bootstrap.cc:654] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.866344  1244 ts_tablet_manager.cc:1918] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:27.872769   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:27.873982   546 catalog_manager.cc:6652] ReplaceTablet: tablet 46ff641001cd4b01a4c09c0d8b79ebf3 deleted and replaced by tablet 5febe562643048f083e579c5f7ba0793
I20250628 07:52:27.875691   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:27.877678  1244 log.cc:1199] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/e4e02676d24a4345908dbbc42fc9c1ab
I20250628 07:52:27.878492  1244 ts_tablet_manager.cc:1939] T e4e02676d24a4345908dbbc42fc9c1ab P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:27.874949   818 raft_consensus.cc:2466] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:27.882205  1234 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:27.882555  1234 ts_tablet_manager.cc:1397] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.037s	user 0.010s	sys 0.001s
I20250628 07:52:27.882525   920 leader_election.cc:304] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:27.882879   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:27.883241  1864 raft_consensus.cc:2802] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:27.883455   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet e4e02676d24a4345908dbbc42fc9c1ab (table default.Table_35424882587f46d1be44e2d8984956b7 [id=4583ea04913744f08ab6d135c8405001]) successfully deleted
I20250628 07:52:27.883639  1864 raft_consensus.cc:695] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
W20250628 07:52:27.884065   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 2ae890710f7f44cf899dd19e96fa9181 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2ae890710f7f44cf899dd19e96fa9181 already in progress: creating tablet
W20250628 07:52:27.884609   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet cc0bf0edebb140209531ccf917fd1ed8 already in progress: creating tablet
I20250628 07:52:27.884375  1864 consensus_queue.cc:237] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.884543  1234 raft_consensus.cc:357] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.885128  1234 raft_consensus.cc:383] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:27.885145   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 46ff641001cd4b01a4c09c0d8b79ebf3 already in progress: creating tablet
I20250628 07:52:27.885356  1234 raft_consensus.cc:738] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:27.892493   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c6c0d1b4e3f549e6a1f103267d886ee9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:27.893324  1234 consensus_queue.cc:260] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:27.894177   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:27.897938  1234 ts_tablet_manager.cc:1428] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.015s	user 0.002s	sys 0.004s
I20250628 07:52:27.898751  1234 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:27.863327  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef5d1e5d358e424d818afeeebab3cfa2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:27.899902  1125 raft_consensus.cc:3058] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:27.906874  1125 raft_consensus.cc:2466] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:27.907830  1229 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:27.889863  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:27.908370  1234 tablet_bootstrap.cc:654] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:27.909240   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 46ff641001cd4b01a4c09c0d8b79ebf3 already in progress: creating tablet
I20250628 07:52:27.916673  1230 ts_tablet_manager.cc:1428] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.253s	user 0.005s	sys 0.000s
I20250628 07:52:27.924089  1866 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.924467  1866 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.926272  1866 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.927678   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:27.928164   818 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:27.929406   920 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:27.930049  1866 raft_consensus.cc:2802] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:52:27.930328  1866 raft_consensus.cc:491] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:27.930645  1866 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:27.938385  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:27.938997  1125 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:27.942050  1863 leader_election.cc:290] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:27.948472  1801 raft_consensus.cc:491] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:27.948858  1801 raft_consensus.cc:513] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.950826  1801 leader_election.cc:290] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:27.952514   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:27.952771  1230 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:27.953725  1062 leader_election.cc:343] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:27.978605  1229 tablet_bootstrap.cc:654] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:27.978986   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:27.981700   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d3f94717f2e3408abde6e223f1920e54" } modify_external_catalogs: true
I20250628 07:52:27.982239   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d3f94717f2e3408abde6e223f1920e54" } modify_external_catalogs: true
W20250628 07:52:27.993439  1062 leader_election.cc:343] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:27.993386  1866 raft_consensus.cc:513] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:27.993762  1062 leader_election.cc:304] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:27.994700  1801 raft_consensus.cc:2747] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:27.995620   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:27.996208   818 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:27.998831  1866 leader_election.cc:290] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.003504   818 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 2.
I20250628 07:52:28.006145   920 leader_election.cc:304] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:28.007367  1865 raft_consensus.cc:2802] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Leader election won for term 2
I20250628 07:52:28.012991  1865 raft_consensus.cc:695] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:28.014117  1865 consensus_queue.cc:237] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:28.019268  1895 log.cc:927] Time spent T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Append to log took a long time: real 0.100s	user 0.001s	sys 0.000s
I20250628 07:52:28.026698  1801 raft_consensus.cc:491] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.026850  1229 tablet_bootstrap.cc:492] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:28.027184  1229 ts_tablet_manager.cc:1397] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.120s	user 0.007s	sys 0.007s
I20250628 07:52:28.027081  1801 raft_consensus.cc:513] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.031675  1801 leader_election.cc:290] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:28.031924   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:28.033054  1062 leader_election.cc:343] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.036406   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:28.037593  1062 leader_election.cc:343] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.037942  1062 leader_election.cc:304] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.038983  1897 raft_consensus.cc:2747] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:28.046330  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "71da5208ceef4186855cbe471f0fd8a3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.047082  1125 raft_consensus.cc:3058] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:28.075358  1863 raft_consensus.cc:491] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.075835  1863 raft_consensus.cc:513] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.096839   545 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet c8c9bec0ff3e48c98e91157d1374ce4e
I20250628 07:52:28.118757  1234 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.119168  1234 ts_tablet_manager.cc:1397] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.221s	user 0.024s	sys 0.037s
I20250628 07:52:28.119284   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9a248bf8a62b4b4eb206c0daff5e0568" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:28.119866   818 raft_consensus.cc:2466] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:28.120232  1229 raft_consensus.cc:357] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.120882   920 leader_election.cc:304] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
W20250628 07:52:28.121354  1061 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.Ping -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 56958.2us
I20250628 07:52:28.121606  1864 raft_consensus.cc:2802] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.121922  1864 raft_consensus.cc:491] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.121869  1234 raft_consensus.cc:357] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.122511  1234 raft_consensus.cc:383] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.122771  1234 raft_consensus.cc:738] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.122215  1864 raft_consensus.cc:3058] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.129384  1864 raft_consensus.cc:513] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.133322  1234 consensus_queue.cc:260] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.121052  1229 raft_consensus.cc:383] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.135901  1229 raft_consensus.cc:738] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.136153  1234 ts_tablet_manager.cc:1428] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.001s
I20250628 07:52:28.137039  1229 consensus_queue.cc:260] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.137840  1234 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:28.140473   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9a248bf8a62b4b4eb206c0daff5e0568" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:28.140928   818 raft_consensus.cc:3058] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.148186   818 raft_consensus.cc:2466] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:28.149187   920 leader_election.cc:304] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:28.150058  1234 tablet_bootstrap.cc:654] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.150081  1863 leader_election.cc:290] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.154796  1865 raft_consensus.cc:2802] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.156420  1229 ts_tablet_manager.cc:1428] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.129s	user 0.006s	sys 0.001s
I20250628 07:52:28.159101  1864 leader_election.cc:290] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:28.161512   921 leader_election.cc:343] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:28.160744  1865 raft_consensus.cc:695] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:28.165920  1865 consensus_queue.cc:237] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.171097  1229 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:28.176044  1229 tablet_bootstrap.cc:654] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.177201  1234 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.178012  1234 ts_tablet_manager.cc:1397] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.040s	user 0.011s	sys 0.001s
I20250628 07:52:28.180244  1234 raft_consensus.cc:357] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.180868  1234 raft_consensus.cc:383] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.181102  1234 raft_consensus.cc:738] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.181727  1234 consensus_queue.cc:260] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:28.182132   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89], current schema version=3) failed: Not found: failed to reset TS proxy: No leader found: Tablet 8feee3be1a664cae9da417b190163a94 consensus state has no leader
I20250628 07:52:28.183676  1234 ts_tablet_manager.cc:1428] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:28.184885  1234 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:28.192176  1875 raft_consensus.cc:491] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.192513  1875 raft_consensus.cc:513] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.194029  1875 leader_election.cc:290] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.195906  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:28.196143   543 master_service.cc:948] ReplaceTablet: received request to replace tablet 5475a89692eb450d82cde521ec8c9a73 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:28.196431  1128 raft_consensus.cc:2466] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.196854   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.197306   758 leader_election.cc:304] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
W20250628 07:52:28.201314   757 leader_election.cc:343] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.202093  1875 raft_consensus.cc:2802] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.202407  1875 raft_consensus.cc:491] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.202710  1875 raft_consensus.cc:3058] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.203446  1229 tablet_bootstrap.cc:492] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:28.204985  1229 ts_tablet_manager.cc:1397] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.048s	user 0.009s	sys 0.000s
I20250628 07:52:28.214489  1234 tablet_bootstrap.cc:654] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:28.219566   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 49129.2us
I20250628 07:52:28.220778  1230 tablet_bootstrap.cc:654] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.208528  1229 raft_consensus.cc:357] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.221772  1229 raft_consensus.cc:383] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.222046  1229 raft_consensus.cc:738] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.222070  1875 raft_consensus.cc:513] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.224323  1229 consensus_queue.cc:260] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.225664   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:28.226250   986 raft_consensus.cc:3058] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.231020  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9ba2a10acdf0462bb05628351cdbbbe4" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.231583  1128 raft_consensus.cc:3058] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.239132  1128 raft_consensus.cc:2466] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.240126   758 leader_election.cc:304] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:28.245872  1875 leader_election.cc:290] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.246227  1229 ts_tablet_manager.cc:1428] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.040s	user 0.005s	sys 0.000s
I20250628 07:52:28.246397  1875 raft_consensus.cc:2802] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.246793  1875 raft_consensus.cc:695] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.247591  1875 consensus_queue.cc:237] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.249240   986 raft_consensus.cc:2466] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
W20250628 07:52:28.250885   921 leader_election.cc:343] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:28.251061  1229 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:28.254765  1234 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.255148  1234 ts_tablet_manager.cc:1397] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.070s	user 0.007s	sys 0.000s
I20250628 07:52:28.263160  1234 raft_consensus.cc:357] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.263891  1234 raft_consensus.cc:383] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.264117  1234 raft_consensus.cc:738] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.265285  1234 consensus_queue.cc:260] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.274747  1234 ts_tablet_manager.cc:1428] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.019s	user 0.006s	sys 0.000s
I20250628 07:52:28.278048  1234 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:28.290753  1234 tablet_bootstrap.cc:654] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.290899  1807 ts_tablet_manager.cc:1918] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:28.291311  1807 log.cc:1199] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fab31b073e2e49318b4118851f81b6d2
I20250628 07:52:28.291960  1807 ts_tablet_manager.cc:1939] T fab31b073e2e49318b4118851f81b6d2 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:28.139075  1125 raft_consensus.cc:2466] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 2.
I20250628 07:52:28.305753  1911 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.309762  1234 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.310166  1234 ts_tablet_manager.cc:1397] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.000s
I20250628 07:52:28.311304  1910 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.314867  1916 tablet.cc:1722] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.318233  1234 raft_consensus.cc:357] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.318912  1234 raft_consensus.cc:383] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.319165  1234 raft_consensus.cc:738] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.319820  1234 consensus_queue.cc:260] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.330497  1229 tablet_bootstrap.cc:654] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.329130  1234 ts_tablet_manager.cc:1428] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.018s	user 0.004s	sys 0.000s
I20250628 07:52:28.339607  1234 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:28.346761  1230 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:28.347220  1230 ts_tablet_manager.cc:1397] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.395s	user 0.007s	sys 0.006s
I20250628 07:52:28.347707  1234 tablet_bootstrap.cc:654] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.354585  1230 raft_consensus.cc:357] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.355155  1230 raft_consensus.cc:383] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.355430  1230 raft_consensus.cc:738] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.365343  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 221) took 2335 ms. Trace:
I20250628 07:52:28.365711  1807 rpcz_store.cc:276] 0628 07:52:26.030387 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.030803 (+   416us) service_pool.cc:225] Handling call
0628 07:52:27.438939 (+1408136us) ts_tablet_manager.cc:1145] Deleting tablet fab31b073e2e49318b4118851f81b6d2
0628 07:52:27.439022 (+    83us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.023326 (+584304us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:28.290642 (+267316us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:28.365252 (+ 74610us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1309959}
I20250628 07:52:28.367673   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fab31b073e2e49318b4118851f81b6d2 (table default.Table_1935e83524ad453c8175710332a3c459 [id=491d07dd82d54928b39605ba6d28af42]) successfully deleted
I20250628 07:52:28.370781  1234 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.371196  1234 ts_tablet_manager.cc:1397] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.032s	user 0.006s	sys 0.003s
I20250628 07:52:28.373587   539 master_service.cc:948] ReplaceTablet: received request to replace tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:28.373433  1234 raft_consensus.cc:357] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.374068  1234 raft_consensus.cc:383] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.374307  1234 raft_consensus.cc:738] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.377053  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 225) took 2236 ms. Trace:
I20250628 07:52:28.377404  1807 rpcz_store.cc:276] 0628 07:52:26.140343 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.140483 (+   140us) service_pool.cc:225] Handling call
0628 07:52:28.376605 (+2236122us) ts_tablet_manager.cc:1145] Deleting tablet b2d49e5a06384758bce5e7328b6a10f8
0628 07:52:28.376666 (+    61us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.376998 (+   332us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2159277}
I20250628 07:52:28.381737  1234 consensus_queue.cc:260] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:28.382529   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet b2d49e5a06384758bce5e7328b6a10f8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b2d49e5a06384758bce5e7328b6a10f8 already in progress: creating tablet
I20250628 07:52:28.382943  1807 tablet_replica.cc:331] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:28.392486   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6541827cef9a442685f4b25c2539eb33"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:28.393777  1234 ts_tablet_manager.cc:1428] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.022s	user 0.005s	sys 0.000s
W20250628 07:52:28.394250   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6541827cef9a442685f4b25c2539eb33 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:28.396947  1234 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:28.402052  1875 raft_consensus.cc:491] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.402478  1875 raft_consensus.cc:513] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.402844  1866 raft_consensus.cc:491] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.403388  1866 raft_consensus.cc:513] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.404268  1875 leader_election.cc:290] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.405881  1807 raft_consensus.cc:2241] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:28.406018   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.406364  1807 raft_consensus.cc:2270] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:28.406909  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:28.407147   757 leader_election.cc:343] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.406635  1234 tablet_bootstrap.cc:654] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.407511  1127 raft_consensus.cc:2466] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:28.408188   921 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:28.408344  1807 ts_tablet_manager.cc:1905] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:28.407517  1866 leader_election.cc:290] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.408492   758 leader_election.cc:304] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.409554  1875 raft_consensus.cc:2802] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.409879  1875 raft_consensus.cc:491] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.410167  1875 raft_consensus.cc:3058] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.413853  1908 raft_consensus.cc:491] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.414320  1908 raft_consensus.cc:513] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.416064  1908 leader_election.cc:290] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.417454   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:28.418576   757 leader_election.cc:343] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.418980   819 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f29c8ffb7df410295278eee9c578b03" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:28.419098  1875 raft_consensus.cc:513] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.419580   819 raft_consensus.cc:2466] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:28.419831  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:28.420468   920 leader_election.cc:304] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:28.420962  1875 leader_election.cc:290] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.421557   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:28.422173  1866 raft_consensus.cc:2802] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.422282  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7eca25db66fa42a798fc002d43f7e3c3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.422482  1866 raft_consensus.cc:491] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.422780  1866 raft_consensus.cc:3058] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.422811  1128 raft_consensus.cc:3058] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:28.425748   757 leader_election.cc:343] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.426744  1229 tablet_bootstrap.cc:492] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:28.427109  1229 ts_tablet_manager.cc:1397] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.176s	user 0.005s	sys 0.004s
I20250628 07:52:28.429358  1230 consensus_queue.cc:260] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.432065  1128 raft_consensus.cc:2466] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.433195   758 leader_election.cc:304] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.420436  1127 raft_consensus.cc:2466] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.434768   758 leader_election.cc:304] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.436234  1908 raft_consensus.cc:2802] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.440663  1875 raft_consensus.cc:2802] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.441736  1875 raft_consensus.cc:491] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.442710  1875 raft_consensus.cc:3058] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.451937   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_cfe73367b7384124927ff17e68820937" } modify_external_catalogs: true
I20250628 07:52:28.452358   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_cfe73367b7384124927ff17e68820937" } modify_external_catalogs: true
I20250628 07:52:28.454123  1910 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.456981  1926 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.470022  1807 ts_tablet_manager.cc:1918] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:28.470472  1807 log.cc:1199] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:28.471256  1807 ts_tablet_manager.cc:1939] T f42b22f7096e46cf9cc09d1caecc9efb P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:28.471913  1230 ts_tablet_manager.cc:1428] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.124s	user 0.003s	sys 0.002s
I20250628 07:52:28.473028  1230 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:28.485888   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a374b2114f7447f392f342b445cd482d" } new_table_name: "default.Table_478b27f6d90d4be39f296aa259fd334e" modify_external_catalogs: true
I20250628 07:52:28.489435  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 229) took 1845 ms. Trace:
I20250628 07:52:28.489868  1807 rpcz_store.cc:276] 0628 07:52:26.643569 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.643691 (+   122us) service_pool.cc:225] Handling call
0628 07:52:28.382750 (+1739059us) ts_tablet_manager.cc:1145] Deleting tablet f42b22f7096e46cf9cc09d1caecc9efb
0628 07:52:28.382808 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.436183 (+ 53375us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:28.469718 (+ 33535us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:28.489360 (+ 19642us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1737947}
I20250628 07:52:28.491138   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet f42b22f7096e46cf9cc09d1caecc9efb (table default.Table_d3f94717f2e3408abde6e223f1920e54 [id=ef817f7a61734cd1b0d878e43eed8e91]) successfully deleted
I20250628 07:52:28.494235  1229 raft_consensus.cc:357] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.494980  1229 raft_consensus.cc:383] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.495183  1229 raft_consensus.cc:738] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.495703  1229 consensus_queue.cc:260] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.501758  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 231) took 1806 ms. Trace:
I20250628 07:52:28.502053  1807 rpcz_store.cc:276] 0628 07:52:26.694997 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:26.695116 (+   119us) service_pool.cc:225] Handling call
0628 07:52:28.497353 (+1802237us) ts_tablet_manager.cc:1145] Deleting tablet f845f32699044867af6f0f2b655cad8e
0628 07:52:28.497835 (+   482us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.501704 (+  3869us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":39680,"tablet-delete.queue_time_us":1801750}
W20250628 07:52:28.503046   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet f845f32699044867af6f0f2b655cad8e because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f845f32699044867af6f0f2b655cad8e already in progress: creating tablet
I20250628 07:52:28.503383  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 241) took 1075 ms. Trace:
I20250628 07:52:28.503679  1807 rpcz_store.cc:276] 0628 07:52:27.428237 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:27.461325 (+ 33088us) service_pool.cc:225] Handling call
0628 07:52:28.503094 (+1041769us) ts_tablet_manager.cc:1145] Deleting tablet 2ae890710f7f44cf899dd19e96fa9181
0628 07:52:28.503129 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.503333 (+   204us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1041009}
I20250628 07:52:28.505785  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 243) took 1036 ms. Trace:
W20250628 07:52:28.506275   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 2ae890710f7f44cf899dd19e96fa9181 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 2ae890710f7f44cf899dd19e96fa9181 already in progress: creating tablet
I20250628 07:52:28.508579  1908 raft_consensus.cc:695] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.506078  1807 rpcz_store.cc:276] 0628 07:52:27.469475 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:27.529160 (+ 59685us) service_pool.cc:225] Handling call
0628 07:52:28.505483 (+976323us) ts_tablet_manager.cc:1145] Deleting tablet cc0bf0edebb140209531ccf917fd1ed8
0628 07:52:28.505520 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:28.505741 (+   221us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":975552}
W20250628 07:52:28.510262   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet cc0bf0edebb140209531ccf917fd1ed8 already in progress: creating tablet
I20250628 07:52:28.511234  1908 consensus_queue.cc:237] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:28.513785   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 46ff641001cd4b01a4c09c0d8b79ebf3 already in progress: creating tablet
I20250628 07:52:28.514454  1928 raft_consensus.cc:491] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.515378  1928 raft_consensus.cc:513] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.519472  1866 raft_consensus.cc:513] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.526350  1229 ts_tablet_manager.cc:1428] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.099s	user 0.005s	sys 0.000s
I20250628 07:52:28.533499  1875 raft_consensus.cc:513] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.541705  1925 raft_consensus.cc:491] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.563985   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet e90e850036444335a7e67b383b7e6e4b
I20250628 07:52:28.567929  1866 leader_election.cc:290] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:28.568852   921 leader_election.cc:343] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:28.569478  1229 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:28.573287  1230 tablet_bootstrap.cc:654] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:18.957446  1485 debug-util.cc:398] Leaking SignalData structure 0x7b0800000a80 after lost signal to thread 1384
W20250628 07:52:28.576210  1485 kernel_stack_watchdog.cc:198] Thread 1384 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 387ms:
Kernel stack:
(could not read kernel stack)
User stack:
<Timed out: thread did not respond: maybe it is blocking signals>
W20250628 07:52:17.594848  1487 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:28.584291   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0f29c8ffb7df410295278eee9c578b03" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:28.584898   818 raft_consensus.cc:3058] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.595923   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:28.597095   543 catalog_manager.cc:6652] ReplaceTablet: tablet 5475a89692eb450d82cde521ec8c9a73 deleted and replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df
I20250628 07:52:28.600589   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7dc8fa39558f4514bbe045fbd9c99910" } new_table_name: "default.Table_d81430dc148d442e8dbfc9029c7cc3c1" modify_external_catalogs: true
I20250628 07:52:28.601728   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:28.575237  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:28.603508  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:28.603915  1127 raft_consensus.cc:2466] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:28.604511   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 5475a89692eb450d82cde521ec8c9a73 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5475a89692eb450d82cde521ec8c9a73 already in progress: creating tablet
I20250628 07:52:28.605643  1934 tablet_replica.cc:331] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:28.606595  1934 raft_consensus.cc:2241] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:28.607867  1934 raft_consensus.cc:2270] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:28.611960  1934 ts_tablet_manager.cc:1905] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:28.617723   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d9f1fae1f46f45efbde92d35775fdd0e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:28.619078   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d9f1fae1f46f45efbde92d35775fdd0e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:28.636919   540 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 66353977776545d4a712f58db2aff108
I20250628 07:52:28.640128  1230 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:28.640488  1230 ts_tablet_manager.cc:1397] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.168s	user 0.004s	sys 0.008s
I20250628 07:52:28.642472  1230 raft_consensus.cc:357] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.643079  1230 raft_consensus.cc:383] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.643302  1230 raft_consensus.cc:738] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.643914  1230 consensus_queue.cc:260] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.646510  1230 ts_tablet_manager.cc:1428] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.002s
I20250628 07:52:28.647214  1230 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:28.652175  1230 tablet_bootstrap.cc:654] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.652434   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_04dc0144b3954979b9fdcce449c44782"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:28.653726   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_04dc0144b3954979b9fdcce449c44782 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:28.656744  1863 consensus_queue.cc:1035] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:28.658895  1934 ts_tablet_manager.cc:1918] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:28.659253  1934 log.cc:1199] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:28.659960  1934 ts_tablet_manager.cc:1939] T 5475a89692eb450d82cde521ec8c9a73 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:28.671272   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 5475a89692eb450d82cde521ec8c9a73 (table default.Table_d22b2c7f99204fd68d8a98c993ea9f40 [id=28517a67c8e24b8e9575c49c90edea15]) successfully deleted
W20250628 07:52:28.673008  1384 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 11.105s	user 0.635s	sys 0.900s
W20250628 07:52:28.673614  1384 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 11.106s	user 0.635s	sys 0.900s
I20250628 07:52:28.676120  1230 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:28.676481  1230 ts_tablet_manager.cc:1397] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.029s	user 0.005s	sys 0.004s
I20250628 07:52:28.680636  1230 raft_consensus.cc:357] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.683130  1230 raft_consensus.cc:383] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.683327  1230 raft_consensus.cc:738] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.684427  1230 consensus_queue.cc:260] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.687127  1230 ts_tablet_manager.cc:1428] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.009s	user 0.004s	sys 0.000s
I20250628 07:52:28.687844  1230 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:28.700369   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.701007  1230 tablet_bootstrap.cc:654] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.700889   986 raft_consensus.cc:2466] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.705538   758 leader_election.cc:304] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:28.553153  1928 leader_election.cc:290] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.712294   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:28.597661  1931 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:28.574082  1925 raft_consensus.cc:513] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.715962  1925 leader_election.cc:290] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.717135  1875 leader_election.cc:290] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.722117   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.722601   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:28.722770   983 raft_consensus.cc:2466] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.723104  1928 raft_consensus.cc:491] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.723471  1928 raft_consensus.cc:513] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.725466  1928 leader_election.cc:290] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.725916  1928 raft_consensus.cc:491] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.727609  1932 raft_consensus.cc:2802] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.727922  1932 raft_consensus.cc:491] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.728196  1932 raft_consensus.cc:3058] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.733007  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:28.733690  1127 raft_consensus.cc:2466] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.738750  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.739262  1125 raft_consensus.cc:3058] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.742722   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cb62cf19aaf04444845358cc8ad611fb" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.743185   758 leader_election.cc:304] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:28.744199  1908 raft_consensus.cc:2802] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.744486  1908 raft_consensus.cc:491] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.744786  1908 raft_consensus.cc:3058] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:28.750174   757 leader_election.cc:343] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.769461  1125 raft_consensus.cc:2466] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.771595   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_32928f5c3c044deaa68ad9b7971a3aaf" } modify_external_catalogs: true
I20250628 07:52:28.772797   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_32928f5c3c044deaa68ad9b7971a3aaf" } modify_external_catalogs: true
W20250628 07:52:28.773530   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet d9e0ea5e300347c7adc8e30b9070976b consensus state has no leader
I20250628 07:52:28.776594   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5f97f65bd045458e83a0d56856db2b89"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:28.779955   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5f97f65bd045458e83a0d56856db2b89 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:28.726317  1928 raft_consensus.cc:513] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.788213  1928 leader_election.cc:290] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:28.788551   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5475a89692eb450d82cde521ec8c9a73 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5475a89692eb450d82cde521ec8c9a73 already in progress: creating tablet
I20250628 07:52:28.790819   818 raft_consensus.cc:2466] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:28.791033  1864 consensus_queue.cc:1035] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:28.795714  1384 server_base.cc:1048] running on GCE node
I20250628 07:52:28.792554  1908 raft_consensus.cc:513] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.797912  1932 raft_consensus.cc:513] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:28.785885   757 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.800343  1908 leader_election.cc:290] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.804121  1125 raft_consensus.cc:1273] T 71da5208ceef4186855cbe471f0fd8a3 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20250628 07:52:28.807935   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ef060774191146168ee8d1e13c9cd71b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.814759  1932 leader_election.cc:290] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.817575   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:28.818094   983 raft_consensus.cc:3058] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:28.821713   758 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:28.823647   758 leader_election.cc:304] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.824357  1932 raft_consensus.cc:2747] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:28.825273   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4ed2c883318848a19e2dac4d9820b2e3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:28.825690   757 leader_election.cc:343] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:28.826709   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4ed2c883318848a19e2dac4d9820b2e3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:28.826722   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:28.827174   984 raft_consensus.cc:3058] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.827742   983 raft_consensus.cc:2466] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.830994   920 leader_election.cc:304] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:28.832283   757 leader_election.cc:304] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:28.824776   758 leader_election.cc:304] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.833019  1932 raft_consensus.cc:2802] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.833253  1864 raft_consensus.cc:2802] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.833458  1932 raft_consensus.cc:695] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.834165  1932 consensus_queue.cc:237] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.835618  1864 raft_consensus.cc:695] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:28.836340  1864 consensus_queue.cc:237] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.841481   984 raft_consensus.cc:2466] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.837045  1908 raft_consensus.cc:2802] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.845502  1908 raft_consensus.cc:695] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.841389  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5fa4299a9e0f4b409d04c8c5900901c0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.846206  1908 consensus_queue.cc:237] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.846870  1126 raft_consensus.cc:3058] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.842052  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "36eebc4a46bd454da5bc176df2a9680f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.849375  1129 raft_consensus.cc:3058] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.856349  1897 raft_consensus.cc:491] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.856787  1897 raft_consensus.cc:513] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.858548  1897 leader_election.cc:290] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
W20250628 07:52:28.859786   758 leader_election.cc:343] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:28.860221   758 leader_election.cc:304] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.861243  1928 raft_consensus.cc:2747] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:28.862028   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9579dd44e5e241aea2671be28e16ef7c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:28.863780  1062 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.872241  1129 raft_consensus.cc:2466] T 36eebc4a46bd454da5bc176df2a9680f P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.876919   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9579dd44e5e241aea2671be28e16ef7c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:28.883960   757 leader_election.cc:304] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:28.876093  1126 raft_consensus.cc:2466] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
W20250628 07:52:28.880395  1062 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.884709   821 raft_consensus.cc:1273] T 71da5208ceef4186855cbe471f0fd8a3 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20250628 07:52:28.885000  1062 leader_election.cc:304] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:28.885231  1932 raft_consensus.cc:2802] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.885634  1932 raft_consensus.cc:695] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.886361  1932 consensus_queue.cc:237] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.889050  1863 consensus_queue.cc:1035] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:28.891747  1897 raft_consensus.cc:2747] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:28.900499  1864 consensus_queue.cc:1035] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:28.901489  1234 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:28.901901  1234 ts_tablet_manager.cc:1397] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.505s	user 0.064s	sys 0.092s
W20250628 07:52:28.907896  1938 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:52:28.910269  1384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:52:28.911989  1908 raft_consensus.cc:491] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.912387  1908 raft_consensus.cc:513] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.915233  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:28.915724  1128 raft_consensus.cc:2466] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:28.916564   758 leader_election.cc:304] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:28.917301  1928 raft_consensus.cc:2802] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:28.917585  1928 raft_consensus.cc:491] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:28.917892  1928 raft_consensus.cc:3058] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.918462   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:28.920255   757 leader_election.cc:343] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.925442  1928 raft_consensus.cc:513] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.927132  1908 leader_election.cc:290] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.929400  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:28.929692  1928 leader_election.cc:290] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:28.929893  1128 raft_consensus.cc:3058] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:28.930249  1234 raft_consensus.cc:357] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.930872  1234 raft_consensus.cc:383] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:28.931094  1234 raft_consensus.cc:738] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:28.931612  1234 consensus_queue.cc:260] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.936786  1128 raft_consensus.cc:2466] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:28.937718   758 leader_election.cc:304] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:28.939569  1928 raft_consensus.cc:2802] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:28.940176  1928 raft_consensus.cc:695] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:28.943570  1928 consensus_queue.cc:237] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.951802   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4ba7d2634cb74f6f858a5a3db8493b6a" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:28.956569   757 leader_election.cc:343] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.587880  1229 tablet_bootstrap.cc:654] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:28.961333   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_39092c471b2443d183b7229a1d625b66" } modify_external_catalogs: true
I20250628 07:52:28.962014  1897 raft_consensus.cc:491] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:28.962934   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_39092c471b2443d183b7229a1d625b66" } modify_external_catalogs: true
I20250628 07:52:28.962420  1897 raft_consensus.cc:513] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:28.968174  1234 ts_tablet_manager.cc:1428] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.066s	user 0.005s	sys 0.001s
I20250628 07:52:28.968680  1897 leader_election.cc:290] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:28.974980   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:28.981873  1234 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:28.983249  1062 leader_election.cc:343] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:28.988821   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:29.004530  1062 leader_election.cc:343] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.004913  1062 leader_election.cc:304] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:29.005959  1897 raft_consensus.cc:2747] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:29.008764  1866 consensus_queue.cc:1035] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:29.027707  1384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:52:29.032231  1230 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:29.032660  1230 ts_tablet_manager.cc:1397] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.345s	user 0.024s	sys 0.067s
I20250628 07:52:29.034943  1230 raft_consensus.cc:357] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.035607  1230 raft_consensus.cc:383] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.035887  1230 raft_consensus.cc:738] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.035790  1384 hybrid_clock.cc:648] HybridClock initialized: now 1751097149035654 us; error 135 us; skew 500 ppm
I20250628 07:52:29.060905   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]). Expected version 3 got 0
I20250628 07:52:29.061375  1230 consensus_queue.cc:260] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.064116  1230 ts_tablet_manager.cc:1428] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.031s	user 0.006s	sys 0.001s
I20250628 07:52:29.064963  1230 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:29.092010   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1fe54aa8c2a04c238afb82ef9ff7fa81
I20250628 07:52:29.088852  1230 tablet_bootstrap.cc:654] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:29.093113  1897 raft_consensus.cc:491] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:29.093542  1897 raft_consensus.cc:513] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.094725   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:29.095228  1897 leader_election.cc:290] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:29.096606  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:29.096788   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:29.098094   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 8155990eebd84fcf9ebc357e186d1bf2 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 8155990eebd84fcf9ebc357e186d1bf2 already in progress: creating tablet
I20250628 07:52:29.099947   539 catalog_manager.cc:6652] ReplaceTablet: tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 deleted and replaced by tablet 7986ee140247478eafc2d95aaec8f8b4
I20250628 07:52:29.100097   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b2d49e5a06384758bce5e7328b6a10f8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:29.101231  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 with delete_type TABLET_DATA_DELETED (replaced by tablet 7986ee140247478eafc2d95aaec8f8b4) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:29.101882   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 8155990eebd84fcf9ebc357e186d1bf2 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 8155990eebd84fcf9ebc357e186d1bf2 already in progress: creating tablet
I20250628 07:52:29.103636   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 with delete_type TABLET_DATA_DELETED (replaced by tablet 7986ee140247478eafc2d95aaec8f8b4) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:29.104786  1062 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:29.105608   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 already in progress: creating tablet
I20250628 07:52:29.106225   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c974994f851542aa92291ccc82d531e0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:29.107007   547 catalog_manager.cc:5582] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:29.109298   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.109180   547 catalog_manager.cc:5582] T 71da5208ceef4186855cbe471f0fd8a3 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 2, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 2 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:29.108973   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 with delete_type TABLET_DATA_DELETED (replaced by tablet 7986ee140247478eafc2d95aaec8f8b4) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.111068   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
W20250628 07:52:29.111846   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 already in progress: creating tablet
W20250628 07:52:29.107599   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c974994f851542aa92291ccc82d531e0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.113001   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b2d49e5a06384758bce5e7328b6a10f8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:29.114715  1062 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.115079  1062 leader_election.cc:304] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:29.115969  1897 raft_consensus.cc:2747] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:29.116878   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 8155990eebd84fcf9ebc357e186d1bf2 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 8155990eebd84fcf9ebc357e186d1bf2 already in progress: creating tablet
W20250628 07:52:29.117467   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 already in progress: creating tablet
I20250628 07:52:29.125666   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet a8518c82c188484c8554d3115637f5be
I20250628 07:52:29.136619  1932 consensus_queue.cc:1035] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:29.138518   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:29.139012  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet a8518c82c188484c8554d3115637f5be with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:29.139262   545 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
W20250628 07:52:29.140304   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:29.140729   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet a8518c82c188484c8554d3115637f5be because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a8518c82c188484c8554d3115637f5be already in progress: creating tablet
I20250628 07:52:29.143513   956 tablet_service.cc:1515] Processing DeleteTablet for tablet a8518c82c188484c8554d3115637f5be with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.145835   797 tablet_service.cc:1515] Processing DeleteTablet for tablet a8518c82c188484c8554d3115637f5be with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:29.147472   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet a8518c82c188484c8554d3115637f5be because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a8518c82c188484c8554d3115637f5be already in progress: creating tablet
W20250628 07:52:29.148087   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet a8518c82c188484c8554d3115637f5be because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a8518c82c188484c8554d3115637f5be already in progress: creating tablet
I20250628 07:52:29.163547   546 catalog_manager.cc:5582] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:29.164721   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:29.166849   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:29.167567   546 catalog_manager.cc:5582] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:29.168706   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:29.191606  1932 consensus_queue.cc:1035] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:29.195345  1384 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:52:29.197814   956 tablet_service.cc:1468] Processing CreateTablet for tablet 8a0f311c619443198e33dfef20a3f4a4 (DEFAULT_TABLE table=default.Table_ade87cf74e6b498690118e3541e41383 [id=2ccbb0163b564f39971348853bdbd012]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.199218   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a0f311c619443198e33dfef20a3f4a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.205983   797 tablet_service.cc:1468] Processing CreateTablet for tablet 8a0f311c619443198e33dfef20a3f4a4 (DEFAULT_TABLE table=default.Table_ade87cf74e6b498690118e3541e41383 [id=2ccbb0163b564f39971348853bdbd012]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.207870   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a0f311c619443198e33dfef20a3f4a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.210235   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d22b2c7f99204fd68d8a98c993ea9f40" } new_table_name: "default.Table_bb6804bddac34e8db4751977623c7ab8" modify_external_catalogs: true
I20250628 07:52:29.214078   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_79306d05ef4945db94f6c1fbf588aa9d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:29.217406  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 8a0f311c619443198e33dfef20a3f4a4 (DEFAULT_TABLE table=default.Table_ade87cf74e6b498690118e3541e41383 [id=2ccbb0163b564f39971348853bdbd012]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.217646   957 tablet_service.cc:1468] Processing CreateTablet for tablet a5ac50e627c0470782f96bd77a61f763 (DEFAULT_TABLE table=default.Table_5ad5f13abefc445b8c69ab0be547644d [id=033b1fae1f634663a3efa53e00c858ac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.218348  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a0f311c619443198e33dfef20a3f4a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.218816   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a5ac50e627c0470782f96bd77a61f763. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.220412  1234 tablet_bootstrap.cc:654] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:29.222186   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_79306d05ef4945db94f6c1fbf588aa9d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.222396  1107 tablet_service.cc:1468] Processing CreateTablet for tablet a5ac50e627c0470782f96bd77a61f763 (DEFAULT_TABLE table=default.Table_5ad5f13abefc445b8c69ab0be547644d [id=033b1fae1f634663a3efa53e00c858ac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.223312  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a5ac50e627c0470782f96bd77a61f763. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.226614   801 tablet_service.cc:1468] Processing CreateTablet for tablet a5ac50e627c0470782f96bd77a61f763 (DEFAULT_TABLE table=default.Table_5ad5f13abefc445b8c69ab0be547644d [id=033b1fae1f634663a3efa53e00c858ac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.237957   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a5ac50e627c0470782f96bd77a61f763. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.250779  1230 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:29.251181  1230 ts_tablet_manager.cc:1397] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.186s	user 0.022s	sys 0.011s
I20250628 07:52:29.254266  1230 raft_consensus.cc:357] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.254904  1230 raft_consensus.cc:383] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.255177  1230 raft_consensus.cc:738] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.255812  1230 consensus_queue.cc:260] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.257917  1230 ts_tablet_manager.cc:1428] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.002s
I20250628 07:52:29.258719  1230 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:29.269130  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c (DEFAULT_TABLE table=default.Table_624e66567efa4b4f9ac68e502edbceda [id=71b8cee9087f4b519d6c6fc32ee442f9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.270653  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 296d777dcbfb4d5aa80db78e3f07426c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.280934   797 tablet_service.cc:1468] Processing CreateTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c (DEFAULT_TABLE table=default.Table_624e66567efa4b4f9ac68e502edbceda [id=71b8cee9087f4b519d6c6fc32ee442f9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.282392   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 296d777dcbfb4d5aa80db78e3f07426c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.292516  1932 raft_consensus.cc:491] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:29.292974  1932 raft_consensus.cc:513] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.295523  1932 leader_election.cc:290] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:29.301570  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 77234b63748849b6996c7aa0ca23ef05 (DEFAULT_TABLE table=default.Table_efdb291a75464ca09777b59f822b3ce7 [id=857b654698d2412da5766828f085087d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.302510  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 77234b63748849b6996c7aa0ca23ef05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.310693  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:29.311273  1129 raft_consensus.cc:2466] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:29.312248   758 leader_election.cc:304] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:29.313066  1932 raft_consensus.cc:2802] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:29.313652  1932 raft_consensus.cc:491] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:29.314005  1932 raft_consensus.cc:3058] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:29.320418  1932 raft_consensus.cc:513] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.331576   955 tablet_service.cc:1468] Processing CreateTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c (DEFAULT_TABLE table=default.Table_624e66567efa4b4f9ac68e502edbceda [id=71b8cee9087f4b519d6c6fc32ee442f9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.332655   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 296d777dcbfb4d5aa80db78e3f07426c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.339768   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:29.345214   757 leader_election.cc:343] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.348652   956 tablet_service.cc:1468] Processing CreateTablet for tablet 77234b63748849b6996c7aa0ca23ef05 (DEFAULT_TABLE table=default.Table_efdb291a75464ca09777b59f822b3ce7 [id=857b654698d2412da5766828f085087d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.350255   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 77234b63748849b6996c7aa0ca23ef05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.364316   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:29.367836   757 leader_election.cc:343] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.371680   957 tablet_service.cc:1468] Processing CreateTablet for tablet af28ae2139b2417381808cb4068ddf60 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.372748   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af28ae2139b2417381808cb4068ddf60. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.366832  1229 tablet_bootstrap.cc:492] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:29.374189  1229 ts_tablet_manager.cc:1397] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.805s	user 0.009s	sys 0.002s
I20250628 07:52:29.386684  1230 tablet_bootstrap.cc:654] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:29.399446  1384 net_util.cc:504] Time spent look up canonical hostname for localhost 'dist-test-slave-kfnw': real 0.202s	user 0.001s	sys 0.000s
I20250628 07:52:29.401608   801 tablet_service.cc:1468] Processing CreateTablet for tablet 77234b63748849b6996c7aa0ca23ef05 (DEFAULT_TABLE table=default.Table_efdb291a75464ca09777b59f822b3ce7 [id=857b654698d2412da5766828f085087d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.402697   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 77234b63748849b6996c7aa0ca23ef05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.406702  1229 raft_consensus.cc:357] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.407389  1229 raft_consensus.cc:383] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.407672  1229 raft_consensus.cc:738] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.408329  1229 consensus_queue.cc:260] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.424504  1932 leader_election.cc:290] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:29.447515  1229 ts_tablet_manager.cc:1428] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.070s	user 0.004s	sys 0.001s
I20250628 07:52:29.448782  1234 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:29.452430   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet d9e0ea5e300347c7adc8e30b9070976b consensus state has no leader
I20250628 07:52:29.453814  1230 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:29.449204  1234 ts_tablet_manager.cc:1397] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.468s	user 0.016s	sys 0.016s
I20250628 07:52:29.456831   797 tablet_service.cc:1468] Processing CreateTablet for tablet af28ae2139b2417381808cb4068ddf60 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.457928   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af28ae2139b2417381808cb4068ddf60. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:29.461512   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 2595308cbc0d44f0abfd0a126f16d4df has no consensus state
I20250628 07:52:29.464787  1230 ts_tablet_manager.cc:1397] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.196s	user 0.008s	sys 0.001s
I20250628 07:52:29.463798   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_577ffb94469047e2a14ccb8d89c9a4d6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:29.465715   800 tablet_service.cc:1468] Processing CreateTablet for tablet 01b1767597424753a588090e9228d133 (DEFAULT_TABLE table=default.Table_3eb1f00557094612af6b3582a0ce2679 [id=e491563484064b93b14fe4b3a15c6ebd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.466596   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01b1767597424753a588090e9228d133. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:29.467379   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_577ffb94469047e2a14ccb8d89c9a4d6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.459384  1234 raft_consensus.cc:357] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.471897  1234 raft_consensus.cc:383] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.484642   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:29.500432   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:29.501396   547 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_52441cbec305498ebed774e69004a383 [id=475e75d866ea4a5585227052d39db9da] alter complete (version 2)
I20250628 07:52:29.502840   538 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6] alter complete (version 3)
W20250628 07:52:29.504348   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 63) took 2482 ms (client timeout 1000 ms). Trace:
W20250628 07:52:29.507894   542 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 49) took 2036 ms (client timeout 1000 ms). Trace:
I20250628 07:52:29.508201  1230 raft_consensus.cc:357] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.508842  1230 raft_consensus.cc:383] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.509073  1230 raft_consensus.cc:738] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.509677  1230 consensus_queue.cc:260] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:29.511330   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 56) took 1825 ms (client timeout 1000 ms). Trace:
I20250628 07:52:29.511863  1230 ts_tablet_manager.cc:1428] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.047s	user 0.004s	sys 0.000s
W20250628 07:52:29.511642   544 rpcz_store.cc:269] 0628 07:52:27.685320 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:27.686028 (+   708us) service_pool.cc:225] Handling call
0628 07:52:27.690204 (+  4176us) spinlock_profiling.cc:243] Waited 1.41 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.511163 (+1820959us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":4,"spinlock_wait_cycles":3243904}
I20250628 07:52:29.512676  1230 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:29.472164  1234 raft_consensus.cc:738] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.513653  1234 consensus_queue.cc:260] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.522641   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.523005   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_624e66567efa4b4f9ac68e502edbceda" } modify_external_catalogs: true
I20250628 07:52:29.523577  1940 tablet_replica.cc:331] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:29.524605  1940 raft_consensus.cc:2241] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:29.515758   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 64) took 1479 ms (client timeout 1000 ms). Trace:
I20250628 07:52:29.525800  1940 raft_consensus.cc:2270] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:29.525434   545 rpcz_store.cc:269] 0628 07:52:28.035961 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:28.097396 (+ 61435us) service_pool.cc:225] Handling call
0628 07:52:28.109737 (+ 12341us) spinlock_profiling.cc:243] Waited 7.31 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.515589 (+1405852us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":49088,"reported_tablets":7,"spinlock_wait_cycles":16816512}
I20250628 07:52:29.527088   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_49dd1c44ec2d43b9bdfe6c8169f90406" } modify_external_catalogs: true
I20250628 07:52:29.527794  1940 ts_tablet_manager.cc:1905] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:29.508173   542 rpcz_store.cc:269] 0628 07:52:27.471586 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:27.471728 (+   142us) service_pool.cc:225] Handling call
0628 07:52:27.491003 (+ 19275us) spinlock_profiling.cc:243] Waited 16.7 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.507702 (+2016699us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:28.278851 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:28.279431 (+   580us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:28.279460 (+    29us) write_op.cc:435] Acquired schema lock
0628 07:52:28.279478 (+    18us) tablet.cc:660] Decoding operations
0628 07:52:28.279799 (+   321us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:28.279846 (+    47us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:28.279874 (+    28us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:28.280023 (+   149us) tablet.cc:699] Row locks acquired
0628 07:52:28.280038 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:28.280166 (+   128us) write_op.cc:270] Start()
0628 07:52:28.280254 (+    88us) write_op.cc:276] Timestamp: P: 1751097148280110 usec, L: 0
0628 07:52:28.280275 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:28.298992 (+ 18717us) log.cc:844] Serialized 533 byte log entry
0628 07:52:28.313409 (+ 14417us) spinlock_profiling.cc:243] Waited 18.1 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:28.319687 (+  6278us) op_driver.cc:464] REPLICATION: finished
0628 07:52:28.319941 (+   254us) write_op.cc:301] APPLY: starting
0628 07:52:28.319989 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:28.320066 (+    77us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:28.320083 (+    17us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:28.320249 (+   166us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:28.320363 (+   114us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:28.320397 (+    34us) write_op.cc:312] APPLY: finished
0628 07:52:28.320549 (+   152us) log.cc:844] Serialized 23 byte log entry
0628 07:52:28.320698 (+   149us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.320754 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:28.320783 (+    29us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:28.335987 (+ 15204us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.336010 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":9,"spinlock_wait_cycles":38453504,"child_traces":[["op",{"apply.queue_time_us":88,"apply.run_cpu_time_us":1072,"apply.run_wall_time_us":1072,"num_ops":1,"prepare.queue_time_us":770789,"prepare.run_cpu_time_us":3664,"prepare.run_wall_time_us":57156,"raft.queue_time_us":1404,"raft.run_cpu_time_us":649,"raft.run_wall_time_us":12817,"replication_time_us":39312,"spinlock_wait_cycles":41572096}]]}
I20250628 07:52:29.537317   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d34b80de26354403b4ea3b3994522090"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:29.538519  1234 ts_tablet_manager.cc:1428] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.081s	user 0.001s	sys 0.004s
W20250628 07:52:29.504748   538 rpcz_store.cc:269] 0628 07:52:27.022013 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:27.025422 (+  3409us) service_pool.cc:225] Handling call
0628 07:52:27.067006 (+ 41584us) spinlock_profiling.cc:243] Waited 39.3 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.504258 (+2437252us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:27.394199 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:27.394598 (+   399us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:27.394621 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:27.394633 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:27.394908 (+   275us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:27.394970 (+    62us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:27.394998 (+    28us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:27.395138 (+   140us) tablet.cc:699] Row locks acquired
0628 07:52:27.395152 (+    14us) write_op.cc:260] PREPARE: finished
0628 07:52:27.395262 (+   110us) write_op.cc:270] Start()
0628 07:52:27.395442 (+   180us) write_op.cc:276] Timestamp: P: 1751097147395183 usec, L: 0
0628 07:52:27.395458 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:27.395941 (+   483us) log.cc:844] Serialized 356 byte log entry
0628 07:52:27.407931 (+ 11990us) op_driver.cc:464] REPLICATION: finished
0628 07:52:27.408136 (+   205us) write_op.cc:301] APPLY: starting
0628 07:52:27.408185 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:27.408277 (+    92us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:27.408366 (+    89us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:27.408522 (+   156us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:27.408661 (+   139us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:27.408703 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:27.408861 (+   158us) log.cc:844] Serialized 23 byte log entry
0628 07:52:27.409014 (+   153us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:27.409072 (+    58us) write_op.cc:454] Released schema lock
0628 07:52:27.409604 (+   532us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:27.513242 (+103638us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:27.513266 (+    24us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:28.114115 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:28.114596 (+   481us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:28.114627 (+    31us) write_op.cc:435] Acquired schema lock
0628 07:52:28.114647 (+    20us) tablet.cc:660] Decoding operations
0628 07:52:28.114942 (+   295us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:28.114994 (+    52us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:28.115027 (+    33us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:28.115202 (+   175us) tablet.cc:699] Row locks acquired
0628 07:52:28.115219 (+    17us) write_op.cc:260] PREPARE: finished
0628 07:52:28.115333 (+   114us) write_op.cc:270] Start()
0628 07:52:28.115441 (+   108us) write_op.cc:276] Timestamp: P: 1751097148115219 usec, L: 0
0628 07:52:28.115463 (+    22us) op_driver.cc:348] REPLICATION: starting
0628 07:52:28.116052 (+   589us) log.cc:844] Serialized 356 byte log entry
0628 07:52:28.191466 (+ 75414us) op_driver.cc:464] REPLICATION: finished
0628 07:52:28.192146 (+   680us) write_op.cc:301] APPLY: starting
0628 07:52:28.192194 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:28.192263 (+    69us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:28.192353 (+    90us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:28.192497 (+   144us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:28.192654 (+   157us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:28.192709 (+    55us) write_op.cc:312] APPLY: finished
0628 07:52:28.192885 (+   176us) log.cc:844] Serialized 23 byte log entry
0628 07:52:28.193018 (+   133us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.193075 (+    57us) write_op.cc:454] Released schema lock
0628 07:52:28.193129 (+    54us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:28.193339 (+   210us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.193362 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":4,"spinlock_wait_cycles":90466560,"child_traces":[["op",{"apply.queue_time_us":76,"apply.run_cpu_time_us":1201,"apply.run_wall_time_us":1672,"num_ops":1,"prepare.queue_time_us":278480,"prepare.run_cpu_time_us":1955,"prepare.run_wall_time_us":119003,"raft.queue_time_us":70,"raft.run_cpu_time_us":588,"raft.run_wall_time_us":588,"replication_time_us":12388}],["op",{"apply.queue_time_us":530,"apply.run_cpu_time_us":1177,"apply.run_wall_time_us":1177,"num_ops":1,"prepare.queue_time_us":698325,"prepare.run_cpu_time_us":2288,"prepare.run_wall_time_us":66287,"raft.queue_time_us":3397,"raft.run_cpu_time_us":753,"raft.run_wall_time_us":753,"replication_time_us":75897}]]}
W20250628 07:52:29.541503   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d34b80de26354403b4ea3b3994522090 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:29.547290   757 consensus_peers.cc:489] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:29.548359  1234 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:29.545326   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 280361us
I20250628 07:52:29.573176   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_624e66567efa4b4f9ac68e502edbceda" } modify_external_catalogs: true
I20250628 07:52:29.576514   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_49dd1c44ec2d43b9bdfe6c8169f90406" } modify_external_catalogs: true
I20250628 07:52:29.579770  1230 tablet_bootstrap.cc:654] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:29.582026  1234 tablet_bootstrap.cc:654] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:29.592660  1940 ts_tablet_manager.cc:1918] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:29.593053  1940 log.cc:1199] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:29.593724  1940 ts_tablet_manager.cc:1939] T 0f29c8ffb7df410295278eee9c578b03 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:29.486080   961 tablet_service.cc:1468] Processing CreateTablet for tablet 01b1767597424753a588090e9228d133 (DEFAULT_TABLE table=default.Table_3eb1f00557094612af6b3582a0ce2679 [id=e491563484064b93b14fe4b3a15c6ebd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:29.507910   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.603905   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 0f29c8ffb7df410295278eee9c578b03 (table default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]) successfully deleted
I20250628 07:52:29.508999   963 tablet_service.cc:1515] Processing DeleteTablet for tablet f42b22f7096e46cf9cc09d1caecc9efb with delete_type TABLET_DATA_DELETED (replaced by tablet e90e850036444335a7e67b383b7e6e4b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.605085   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01b1767597424753a588090e9228d133. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:29.545022   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_50f3c0fe4d0445c7b1cc05590d41a738"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:29.610759   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_50f3c0fe4d0445c7b1cc05590d41a738 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.606462  1807 tablet_replica.cc:331] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:29.612103  1807 raft_consensus.cc:2241] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:29.614552   539 master_service.cc:948] ReplaceTablet: received request to replace tablet e28ab08f878d4d4b86a5646f687b08fc from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:29.605037  1384 webserver.cc:469] Webserver started at http://127.0.107.190:42501/ using document root <none> and password file <none>
I20250628 07:52:29.618593  1384 fs_manager.cc:362] Metadata directory not provided
I20250628 07:52:29.618921  1384 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:52:29.619355  1807 raft_consensus.cc:2270] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:29.621627  1229 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:29.622758  1807 ts_tablet_manager.cc:1905] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:29.628842   543 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 312) took 1058 ms. Trace:
I20250628 07:52:29.629166   543 rpcz_store.cc:276] 0628 07:52:28.570529 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:28.597727 (+ 27198us) service_pool.cc:225] Handling call
0628 07:52:28.600486 (+  2759us) spinlock_profiling.cc:243] Waited 2.12 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea9413b6 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc 00007f5be6957206
0628 07:52:28.601208 (+   722us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:28.601880 (+   672us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:28.602491 (+   611us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:28.602692 (+   201us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:28.766518 (+163826us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:29.628650 (+862132us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:28.639498 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:28.640184 (+   686us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:28.640206 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:28.640219 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:28.640436 (+   217us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:28.640466 (+    30us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:28.640487 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:28.640612 (+   125us) tablet.cc:699] Row locks acquired
0628 07:52:28.640624 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:28.640716 (+    92us) write_op.cc:270] Start()
0628 07:52:28.640786 (+    70us) write_op.cc:276] Timestamp: P: 1751097148640684 usec, L: 0
0628 07:52:28.640800 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:28.641276 (+   476us) log.cc:844] Serialized 405 byte log entry
0628 07:52:28.762619 (+121343us) op_driver.cc:464] REPLICATION: finished
0628 07:52:28.763228 (+   609us) write_op.cc:301] APPLY: starting
0628 07:52:28.763283 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:28.763368 (+    85us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:28.763471 (+   103us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:28.763616 (+   145us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:28.763738 (+   122us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:28.763775 (+    37us) write_op.cc:312] APPLY: finished
0628 07:52:28.763939 (+   164us) log.cc:844] Serialized 23 byte log entry
0628 07:52:28.764084 (+   145us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.764140 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:28.764173 (+    33us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:28.764375 (+   202us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:28.764396 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":4879104,"child_traces":[["op",{"apply.queue_time_us":466,"apply.run_cpu_time_us":1119,"apply.run_wall_time_us":1120,"num_ops":1,"prepare.queue_time_us":35844,"prepare.run_cpu_time_us":1728,"prepare.run_wall_time_us":3707,"raft.queue_time_us":1614,"raft.run_cpu_time_us":11,"raft.run_wall_time_us":12,"replication_time_us":121718}]]}
I20250628 07:52:29.648417   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:29.649878   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:29.651921   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.652545  1940 tablet_replica.cc:331] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:29.653378  1940 raft_consensus.cc:2241] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:29.653846  1940 raft_consensus.cc:2270] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:29.656358  1940 ts_tablet_manager.cc:1905] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:29.666860   797 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.669144   797 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.694155  1971 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:29.704907   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 57) took 905 ms (client timeout 1000 ms). Trace:
W20250628 07:52:29.705370   546 rpcz_store.cc:269] 0628 07:52:28.799269 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:28.799385 (+   116us) service_pool.cc:225] Handling call
0628 07:52:29.704707 (+905322us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:29.202383 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:29.202768 (+   385us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:29.202789 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:29.202800 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:29.203075 (+   275us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:29.203111 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:29.203131 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:29.203353 (+   222us) tablet.cc:699] Row locks acquired
0628 07:52:29.203366 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:29.203455 (+    89us) write_op.cc:270] Start()
0628 07:52:29.218283 (+ 14828us) write_op.cc:276] Timestamp: P: 1751097149203418 usec, L: 0
0628 07:52:29.218302 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:29.221103 (+  2801us) log.cc:844] Serialized 913 byte log entry
0628 07:52:29.388793 (+167690us) op_driver.cc:464] REPLICATION: finished
0628 07:52:29.389151 (+   358us) write_op.cc:301] APPLY: starting
0628 07:52:29.389203 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:29.389315 (+   112us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:29.389404 (+    89us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:29.389659 (+   255us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:29.389797 (+   138us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:29.389848 (+    51us) write_op.cc:312] APPLY: finished
0628 07:52:29.390037 (+   189us) log.cc:844] Serialized 29 byte log entry
0628 07:52:29.390161 (+   124us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.390220 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:29.390253 (+    33us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:29.390459 (+   206us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.390483 (+    24us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":22787,"reported_tablets":15,"child_traces":[["op",{"apply.queue_time_us":191,"apply.run_cpu_time_us":1304,"apply.run_wall_time_us":1309,"num_ops":2,"prepare.queue_time_us":7718,"prepare.run_cpu_time_us":1980,"prepare.run_wall_time_us":30073,"raft.queue_time_us":4273,"raft.run_cpu_time_us":602,"raft.run_wall_time_us":1334,"replication_time_us":170399}]]}
I20250628 07:52:29.716775  1230 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:29.717540  1230 ts_tablet_manager.cc:1397] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.205s	user 0.009s	sys 0.002s
I20250628 07:52:29.792546  1970 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:29.796967   757 consensus_peers.cc:489] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:29.801927  1234 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:29.802387  1234 ts_tablet_manager.cc:1397] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.263s	user 0.015s	sys 0.000s
I20250628 07:52:29.804942  1807 ts_tablet_manager.cc:1918] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:29.805382  1807 log.cc:1199] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:29.806422  1807 ts_tablet_manager.cc:1939] T 9a248bf8a62b4b4eb206c0daff5e0568 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:29.818867  1229 tablet_bootstrap.cc:654] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:29.831162   757 consensus_peers.cc:489] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:29.723143   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_58cbee214d3d4a7ab7452f436eb56db3" } new_table_name: "default.Table_a011102f62ad4163af1c9aa371b6b29f" modify_external_catalogs: true
I20250628 07:52:29.807819   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 3519e872e8504a4fb962814e3438bc01
I20250628 07:52:29.838614   545 master_service.cc:948] ReplaceTablet: received request to replace tablet 63115b7be4a94b639c18601d438ed947 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:29.808874   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 9a248bf8a62b4b4eb206c0daff5e0568 (table default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]) successfully deleted
W20250628 07:52:29.840643   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet f42b22f7096e46cf9cc09d1caecc9efb because the tablet was not found. No further retry: Not found: Tablet not found: f42b22f7096e46cf9cc09d1caecc9efb
I20250628 07:52:29.810108   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 296d777dcbfb4d5aa80db78e3f07426c
I20250628 07:52:29.844702   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 3519e872e8504a4fb962814e3438bc01 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.845023  1928 raft_consensus.cc:491] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:29.845449  1928 raft_consensus.cc:513] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.847296  1928 leader_election.cc:290] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:29.848007   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:29.848160   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.848285   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:29.849609   757 leader_election.cc:343] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.854038   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 3519e872e8504a4fb962814e3438bc01 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:29.858064   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 296d777dcbfb4d5aa80db78e3f07426c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 296d777dcbfb4d5aa80db78e3f07426c already in progress: creating tablet
W20250628 07:52:29.859376   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 3519e872e8504a4fb962814e3438bc01 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 3519e872e8504a4fb962814e3438bc01 already in progress: creating tablet
I20250628 07:52:29.864674   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet e28ab08f878d4d4b86a5646f687b08fc
I20250628 07:52:29.865934   539 catalog_manager.cc:6652] ReplaceTablet: tablet e28ab08f878d4d4b86a5646f687b08fc deleted and replaced by tablet 711843da7df74ba2bb22d386bc477a38
I20250628 07:52:29.868475   956 tablet_service.cc:1515] Processing DeleteTablet for tablet e28ab08f878d4d4b86a5646f687b08fc with delete_type TABLET_DATA_DELETED (replaced by tablet 711843da7df74ba2bb22d386bc477a38) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.869262  1807 tablet_replica.cc:331] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:29.869129   797 tablet_service.cc:1515] Processing DeleteTablet for tablet e28ab08f878d4d4b86a5646f687b08fc with delete_type TABLET_DATA_DELETED (replaced by tablet 711843da7df74ba2bb22d386bc477a38) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:29.870007  1807 raft_consensus.cc:2241] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:29.871161  1807 raft_consensus.cc:2270] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:29.872036   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_d81430dc148d442e8dbfc9029c7cc3c1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 1 got 0
I20250628 07:52:29.873165  1807 ts_tablet_manager.cc:1905] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:29.884150   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:29.884689  1229 tablet_bootstrap.cc:492] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:29.885108  1229 ts_tablet_manager.cc:1397] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.264s	user 0.007s	sys 0.001s
I20250628 07:52:29.887095   547 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_a6c8170745a346f9bc81b8e023e95b7c [id=044b1a528f234b7c97a6f7066ad8169b] alter complete (version 2)
I20250628 07:52:29.890033  1234 raft_consensus.cc:357] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.891886  1234 raft_consensus.cc:383] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:29.892129  1234 raft_consensus.cc:738] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:29.894223   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_af13e11b58c24735a4b8dbb9a7fefa22"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:29.895872   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_af13e11b58c24735a4b8dbb9a7fefa22 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.906915   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:29.908147   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 50) took 1379 ms (client timeout 1000 ms). Trace:
I20250628 07:52:29.908775   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ec87d950e26f4664861dc71b072bb347" } new_table_name: "default.Table_ae262f1ee79148c4a8afbabd65d9ac89" modify_external_catalogs: true
I20250628 07:52:29.909422   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:29.928891   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_db0904127a6144d7acb537f17d6891d6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:29.931072   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_db0904127a6144d7acb537f17d6891d6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:29.943035  1940 ts_tablet_manager.cc:1918] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:29.943495  1940 log.cc:1199] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:29.944321  1940 ts_tablet_manager.cc:1939] T 9a248bf8a62b4b4eb206c0daff5e0568 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:29.947122   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 9a248bf8a62b4b4eb206c0daff5e0568 (table default.Table_a148c40376f64b4fb79113972fe5693f [id=3b1b9884c7ac4dbe96dfb3a647f7a2b8]) successfully deleted
I20250628 07:52:29.949156  1234 consensus_queue.cc:260] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.951339  1234 ts_tablet_manager.cc:1428] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.148s	user 0.004s	sys 0.000s
I20250628 07:52:29.952493  1977 raft_consensus.cc:491] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:29.961624  1977 raft_consensus.cc:513] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.947027  1940 tablet_replica.cc:331] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:29.963570  1940 raft_consensus.cc:2241] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:29.965382   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:29.971973  1940 raft_consensus.cc:2270] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:29.972457  1062 leader_election.cc:343] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.968742  1234 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:29.973907  1940 ts_tablet_manager.cc:1905] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:29.974959   546 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 661d92febf1142f99815f70ae9b27edc consensus state has no leader
I20250628 07:52:29.977936   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:29.979382  1977 leader_election.cc:290] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:29.980993  1977 raft_consensus.cc:491] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:29.982133  1062 leader_election.cc:343] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.982946  1062 leader_election.cc:304] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:29.985179  1977 raft_consensus.cc:513] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:29.991400   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:29.992801  1062 leader_election.cc:343] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:29.994423   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 65) took 941 ms (client timeout 1000 ms). Trace:
W20250628 07:52:29.994875   541 rpcz_store.cc:269] 0628 07:52:29.052587 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.099031 (+ 46444us) service_pool.cc:225] Handling call
0628 07:52:29.114525 (+ 15494us) spinlock_profiling.cc:243] Waited 11 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.994347 (+879822us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":14,"spinlock_wait_cycles":25385728}
I20250628 07:52:29.995638   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:29.996812  1062 leader_election.cc:343] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:29.997156  1062 leader_election.cc:304] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:30.001361   920 consensus_peers.cc:489] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:30.006091  1928 consensus_queue.cc:1035] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.014483  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.015699  1105 tablet_service.cc:1515] Processing DeleteTablet for tablet e28ab08f878d4d4b86a5646f687b08fc with delete_type TABLET_DATA_DELETED (replaced by tablet 711843da7df74ba2bb22d386bc477a38) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.014467  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 3519e872e8504a4fb962814e3438bc01 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.017634  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet 296d777dcbfb4d5aa80db78e3f07426c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:29 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.017819  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 01b1767597424753a588090e9228d133 (DEFAULT_TABLE table=default.Table_3eb1f00557094612af6b3582a0ce2679 [id=e491563484064b93b14fe4b3a15c6ebd]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.018505  1104 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.025846  1234 tablet_bootstrap.cc:654] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.019474  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 01b1767597424753a588090e9228d133. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.026964  1940 ts_tablet_manager.cc:1918] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:30.019385  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.028746  1977 leader_election.cc:290] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
W20250628 07:52:30.025657  1107 consensus_meta.cc:287] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.781s	user 0.007s	sys 0.000s
I20250628 07:52:30.029848  1977 raft_consensus.cc:2747] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.022469  1981 raft_consensus.cc:2747] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.032517  1940 log.cc:1199] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:30.036240  1101 consensus_meta.cc:287] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: Time spent flushing consensus metadata: real 0.795s	user 0.005s	sys 0.005s
I20250628 07:52:30.042260  1983 tablet_replica.cc:331] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:30.042479  1940 ts_tablet_manager.cc:1939] T cb62cf19aaf04444845358cc8ad611fb P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:30.050348  1234 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.050760  1234 ts_tablet_manager.cc:1397] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.082s	user 0.008s	sys 0.001s
I20250628 07:52:30.050598  1109 tablet_service.cc:1468] Processing CreateTablet for tablet af28ae2139b2417381808cb4068ddf60 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.051782  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet af28ae2139b2417381808cb4068ddf60. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.052497  1928 consensus_queue.cc:1035] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.053743  1928 consensus_queue.cc:1035] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.053750  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5ec8c1d9386a4922b55df5e06039858d" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:30.056962  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:30.057868  1124 raft_consensus.cc:2466] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:30.054554  1125 raft_consensus.cc:3058] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:30.064348  1229 raft_consensus.cc:357] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.064998  1229 raft_consensus.cc:383] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.052985  1234 raft_consensus.cc:357] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.066499  1234 raft_consensus.cc:383] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.066757  1234 raft_consensus.cc:738] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.067965  1940 tablet_replica.cc:331] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:30.068879  1940 raft_consensus.cc:2241] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:30.069766  1940 raft_consensus.cc:2270] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.072549  1940 ts_tablet_manager.cc:1905] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.082760  1384 fs_manager.cc:714] Time spent opening directory manager: real 0.307s	user 0.006s	sys 0.000s
I20250628 07:52:30.084534  1234 consensus_queue.cc:260] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.087149  1234 ts_tablet_manager.cc:1428] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.036s	user 0.006s	sys 0.000s
I20250628 07:52:30.065259  1229 raft_consensus.cc:738] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.097961  1230 raft_consensus.cc:357] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.098863  1230 raft_consensus.cc:383] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.099175  1230 raft_consensus.cc:738] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.100010  1230 consensus_queue.cc:260] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.046609   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_affcc9f556b943238389e6f1e205745b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:30.103802   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_affcc9f556b943238389e6f1e205745b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.104604  1125 raft_consensus.cc:2466] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:30.106628  1940 ts_tablet_manager.cc:1918] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:30.108033  1230 ts_tablet_manager.cc:1428] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.390s	user 0.003s	sys 0.003s
I20250628 07:52:30.109987  1230 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:30.056246   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_efdb291a75464ca09777b59f822b3ce7" } modify_external_catalogs: true
I20250628 07:52:30.112447   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_efdb291a75464ca09777b59f822b3ce7" } modify_external_catalogs: true
I20250628 07:52:30.070235   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet cb62cf19aaf04444845358cc8ad611fb (table default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]) successfully deleted
I20250628 07:52:30.114603  1940 log.cc:1199] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/ef060774191146168ee8d1e13c9cd71b
W20250628 07:52:30.114837   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 44626.3us
I20250628 07:52:30.115417  1940 ts_tablet_manager.cc:1939] T ef060774191146168ee8d1e13c9cd71b P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:30.116051   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a8b4bbb284774df48f62eda3df2ac372"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:30.117046  1983 raft_consensus.cc:2241] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:30.117445   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a8b4bbb284774df48f62eda3df2ac372 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.117547  1983 raft_consensus.cc:2270] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.130610  1940 tablet_replica.cc:331] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:30.133953  1940 raft_consensus.cc:2241] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:30.127364  1990 raft_consensus.cc:491] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.135675  1990 raft_consensus.cc:513] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.138259  1990 leader_election.cc:290] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:30.139487  1940 raft_consensus.cc:2270] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.142100  1940 ts_tablet_manager.cc:1905] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.143340  1230 tablet_bootstrap.cc:654] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.138373  1234 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:30.149206  1983 ts_tablet_manager.cc:1905] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.149542   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet ef060774191146168ee8d1e13c9cd71b (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
W20250628 07:52:30.155292   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 3519e872e8504a4fb962814e3438bc01 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 3519e872e8504a4fb962814e3438bc01 already in progress: creating tablet
W20250628 07:52:30.155831   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 296d777dcbfb4d5aa80db78e3f07426c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 296d777dcbfb4d5aa80db78e3f07426c already in progress: creating tablet
I20250628 07:52:30.159394   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_5c036fde993047db95ee90d9ffd4be8d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:30.160600  1234 tablet_bootstrap.cc:654] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:30.161165   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5c036fde993047db95ee90d9ffd4be8d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.165298   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5475a89692eb450d82cde521ec8c9a73" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:30.172139   757 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.173449  1990 consensus_queue.cc:1035] T 36eebc4a46bd454da5bc176df2a9680f P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.087754   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9045e51711844332acd5d748d0bbc3af" } modify_external_catalogs: true
I20250628 07:52:30.177387   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9045e51711844332acd5d748d0bbc3af" } modify_external_catalogs: true
I20250628 07:52:30.183082  1940 ts_tablet_manager.cc:1918] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:30.183480  1940 log.cc:1199] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/e28ab08f878d4d4b86a5646f687b08fc
I20250628 07:52:30.184227  1940 ts_tablet_manager.cc:1939] T e28ab08f878d4d4b86a5646f687b08fc P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:30.193325  1230 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:30.193814  1230 ts_tablet_manager.cc:1397] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.084s	user 0.012s	sys 0.000s
I20250628 07:52:30.203235  1234 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.203760  1234 ts_tablet_manager.cc:1397] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.116s	user 0.005s	sys 0.005s
I20250628 07:52:30.185451   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]) successfully deleted
W20250628 07:52:30.209967   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 157961us
I20250628 07:52:30.211589   758 leader_election.cc:304] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.196739  1230 raft_consensus.cc:357] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.217931  1230 raft_consensus.cc:383] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.218207  1230 raft_consensus.cc:738] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.213807  1990 consensus_queue.cc:1035] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.225363  1230 consensus_queue.cc:260] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.216675  1234 raft_consensus.cc:357] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.226110  1990 raft_consensus.cc:2802] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:30.226267  1234 raft_consensus.cc:383] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.226436  1990 raft_consensus.cc:491] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:30.226538  1234 raft_consensus.cc:738] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.226791  1990 raft_consensus.cc:3058] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:30.100145  1229 consensus_queue.cc:260] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.227108  1234 consensus_queue.cc:260] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.229164  1234 ts_tablet_manager.cc:1428] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.025s	user 0.009s	sys 0.000s
I20250628 07:52:30.235462  1230 ts_tablet_manager.cc:1428] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.041s	user 0.007s	sys 0.000s
I20250628 07:52:30.236373  1230 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:30.241539  1990 raft_consensus.cc:513] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.244222   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:30.245419   757 leader_election.cc:343] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.246213  1990 leader_election.cc:290] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:30.248288  1229 ts_tablet_manager.cc:1428] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.363s	user 0.005s	sys 0.000s
I20250628 07:52:30.249548  1229 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:30.256541  1234 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:29.908541   547 rpcz_store.cc:269] 0628 07:52:28.528963 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:28.564421 (+ 35458us) service_pool.cc:225] Handling call
0628 07:52:28.575971 (+ 11550us) spinlock_profiling.cc:243] Waited 9.16 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:29.908087 (+1332116us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:29.117684 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:29.118090 (+   406us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:29.118111 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:29.118125 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:29.118391 (+   266us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:29.118424 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:29.118445 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:29.118675 (+   230us) tablet.cc:699] Row locks acquired
0628 07:52:29.118687 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:29.118791 (+   104us) write_op.cc:270] Start()
0628 07:52:29.118865 (+    74us) write_op.cc:276] Timestamp: P: 1751097149118710 usec, L: 0
0628 07:52:29.118879 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:29.119262 (+   383us) log.cc:844] Serialized 913 byte log entry
0628 07:52:29.132305 (+ 13043us) op_driver.cc:464] REPLICATION: finished
0628 07:52:29.133035 (+   730us) write_op.cc:301] APPLY: starting
0628 07:52:29.133092 (+    57us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:29.133222 (+   130us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:29.133317 (+    95us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:29.133538 (+   221us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:29.133675 (+   137us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:29.133726 (+    51us) write_op.cc:312] APPLY: finished
0628 07:52:29.133896 (+   170us) log.cc:844] Serialized 29 byte log entry
0628 07:52:29.134020 (+   124us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.134085 (+    65us) write_op.cc:454] Released schema lock
0628 07:52:29.134115 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:29.134320 (+   205us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.134355 (+    35us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:29.276991 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:29.277374 (+   383us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:29.277396 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:29.277415 (+    19us) tablet.cc:660] Decoding operations
0628 07:52:29.280685 (+  3270us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:29.280721 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:29.280744 (+    23us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:29.280877 (+   133us) tablet.cc:699] Row locks acquired
0628 07:52:29.280888 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:29.280988 (+   100us) write_op.cc:270] Start()
0628 07:52:29.281061 (+    73us) write_op.cc:276] Timestamp: P: 1751097149280938 usec, L: 0
0628 07:52:29.281077 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:29.281611 (+   534us) log.cc:844] Serialized 356 byte log entry
0628 07:52:29.396041 (+114430us) op_driver.cc:464] REPLICATION: finished
0628 07:52:29.396338 (+   297us) write_op.cc:301] APPLY: starting
0628 07:52:29.396393 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:29.396473 (+    80us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:29.396578 (+   105us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:29.396708 (+   130us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:29.396828 (+   120us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:29.396870 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:29.397011 (+   141us) log.cc:844] Serialized 23 byte log entry
0628 07:52:29.397160 (+   149us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.397209 (+    49us) write_op.cc:454] Released schema lock
0628 07:52:29.397237 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:29.397420 (+   183us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.397445 (+    25us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:29.639863 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:29.640237 (+   374us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:29.640259 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:29.640273 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:29.640480 (+   207us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:29.640516 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:29.640545 (+    29us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:29.640705 (+   160us) tablet.cc:699] Row locks acquired
0628 07:52:29.640732 (+    27us) write_op.cc:260] PREPARE: finished
0628 07:52:29.640824 (+    92us) write_op.cc:270] Start()
0628 07:52:29.640944 (+   120us) write_op.cc:276] Timestamp: P: 1751097149640742 usec, L: 0
0628 07:52:29.640962 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:29.641438 (+   476us) log.cc:844] Serialized 356 byte log entry
0628 07:52:29.877783 (+236345us) op_driver.cc:464] REPLICATION: finished
0628 07:52:29.878019 (+   236us) write_op.cc:301] APPLY: starting
0628 07:52:29.878076 (+    57us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:29.878166 (+    90us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:29.878262 (+    96us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:29.878452 (+   190us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:29.878581 (+   129us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:29.878631 (+    50us) write_op.cc:312] APPLY: finished
0628 07:52:29.878801 (+   170us) log.cc:844] Serialized 23 byte log entry
0628 07:52:29.879372 (+   571us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.879431 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:29.879464 (+    33us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":39653,"reported_tablets":12,"spinlock_wait_cycles":21075328,"child_traces":[["op",{"apply.queue_time_us":561,"apply.run_cpu_time_us":1276,"apply.run_wall_time_us":1276,"mutex_wait_us":60,"num_ops":2,"prepare.queue_time_us":4742,"prepare.run_cpu_time_us":1735,"prepare.run_wall_time_us":1831,"replication_time_us":13331}],["op",{"apply.queue_time_us":168,"apply.run_cpu_time_us":1047,"apply.run_wall_time_us":1053,"num_ops":1,"prepare.queue_time_us":42762,"prepare.run_cpu_time_us":1892,"prepare.run_wall_time_us":19152,"raft.queue_time_us":7760,"raft.run_cpu_time_us":613,"raft.run_wall_time_us":1265,"replication_time_us":114858}],["op",{"apply.queue_time_us":93,"apply.run_cpu_time_us":1211,"apply.run_wall_time_us":1627,"mutex_wait_us":4252,"num_ops":1,"prepare.queue_time_us":4361,"prepare.run_cpu_time_us":1791,"prepare.run_wall_time_us":2005,"raft.queue_time_us":86,"raft.run_cpu_time_us":34,"raft.run_wall_time_us":34,"replication_time_us":236726}]]}
I20250628 07:52:30.203478   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_f892200f501c448a90964969f585547c" } new_table_name: "default.Table_df287bcea5bd414ebe5aee1689733f8c" modify_external_catalogs: true
I20250628 07:52:30.274631  1865 raft_consensus.cc:491] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.275187  1865 raft_consensus.cc:513] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:30.279006   920 leader_election.cc:343] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:30.279917  1865 leader_election.cc:290] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:30.246820   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_239771785031473096dd9da66d0e09d5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:30.288272   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_239771785031473096dd9da66d0e09d5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.298907  1998 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:52:30.308984  1384 fs_manager.cc:730] Time spent opening block manager: real 0.041s	user 0.004s	sys 0.000s
I20250628 07:52:30.309471  1384 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
uuid: "5fd67221fa5f424ea0f83e582629d276"
format_stamp: "Formatted at 2025-06-28 07:51:58 on dist-test-slave-kfnw"
W20250628 07:52:30.319967   921 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:30.321707  1384 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:52:30.325803   547 master_service.cc:948] ReplaceTablet: received request to replace tablet a447edf0960c4739930450a8c24ff6f7 from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:30.329567   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 105488us
I20250628 07:52:30.330579  1987 consensus_queue.cc:1035] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.333250  1987 consensus_queue.cc:1035] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:30.335014   758 leader_election.cc:304] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.335851  1987 raft_consensus.cc:2802] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:30.336547  1987 raft_consensus.cc:695] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:30.337399  1987 consensus_queue.cc:237] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:30.344635   758 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:30.345108   758 leader_election.cc:304] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.345742  1987 raft_consensus.cc:2747] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.348294  1230 tablet_bootstrap.cc:654] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:30.349009  1214 log.cc:927] Time spent T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Append to log took a long time: real 0.064s	user 0.001s	sys 0.000s
W20250628 07:52:30.348534  1209 log.cc:927] Time spent T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Append to log took a long time: real 0.075s	user 0.001s	sys 0.000s
I20250628 07:52:30.351397  1234 tablet_bootstrap.cc:654] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.354197  1983 ts_tablet_manager.cc:1918] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:30.354605  1983 log.cc:1199] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:30.355414  1983 ts_tablet_manager.cc:1939] T 9579dd44e5e241aea2671be28e16ef7c P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:30.358958  1983 tablet_replica.cc:331] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:30.360110  1807 ts_tablet_manager.cc:1918] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:30.361539   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 9579dd44e5e241aea2671be28e16ef7c (table default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]) successfully deleted
I20250628 07:52:30.365579  1987 raft_consensus.cc:491] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.366088  1987 raft_consensus.cc:513] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.369048  1987 leader_election.cc:290] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:30.382915  1983 raft_consensus.cc:2241] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:30.383605  1983 raft_consensus.cc:2270] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.386802  1983 ts_tablet_manager.cc:1905] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.398741  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:30.399288  1126 raft_consensus.cc:3058] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:30.406894  1230 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:30.406641  1126 raft_consensus.cc:2466] T 132cd82d39e84e85a37f21ae208fd165 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:30.409637  1230 ts_tablet_manager.cc:1397] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.173s	user 0.005s	sys 0.015s
I20250628 07:52:30.418205  1230 raft_consensus.cc:357] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.418843  1230 raft_consensus.cc:383] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.419090  1230 raft_consensus.cc:738] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.419894  1230 consensus_queue.cc:260] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.423842  1230 ts_tablet_manager.cc:1428] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.001s
I20250628 07:52:30.428241  1230 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:30.430354   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 950fea8f6bfb409f8868fe5e88d3dddb
I20250628 07:52:30.434819   801 tablet_service.cc:1515] Processing DeleteTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:30.435636  1940 tablet_replica.cc:331] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:30.436524  1940 raft_consensus.cc:2241] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:30.437446  1940 raft_consensus.cc:2270] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.439450  1940 ts_tablet_manager.cc:1905] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.449352  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.453148  1897 raft_consensus.cc:491] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.453577  1897 raft_consensus.cc:513] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.455370  1897 leader_election.cc:290] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:30.454670  1983 ts_tablet_manager.cc:1918] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:30.456912  1983 log.cc:1199] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/e28ab08f878d4d4b86a5646f687b08fc
I20250628 07:52:30.457901   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:30.459208  1062 leader_election.cc:343] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.458727  1230 tablet_bootstrap.cc:654] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.461064  1983 ts_tablet_manager.cc:1939] T e28ab08f878d4d4b86a5646f687b08fc P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:30.466037   758 leader_election.cc:304] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.467342  1990 raft_consensus.cc:2802] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:30.467751  1990 raft_consensus.cc:695] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:30.468568  1990 consensus_queue.cc:237] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.474839   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]) successfully deleted
W20250628 07:52:30.478312   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 3519e872e8504a4fb962814e3438bc01 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 3519e872e8504a4fb962814e3438bc01 already in progress: creating tablet
W20250628 07:52:30.482066   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 296d777dcbfb4d5aa80db78e3f07426c because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 296d777dcbfb4d5aa80db78e3f07426c already in progress: creating tablet
I20250628 07:52:30.484046  1983 tablet_replica.cc:331] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:30.489908  1983 raft_consensus.cc:2241] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:30.493132  1983 raft_consensus.cc:2270] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.506126   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c97313f05be54e4da2bc4cbbc04c8682"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:30.513553  1983 ts_tablet_manager.cc:1905] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:30.520453   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 54509.4us
I20250628 07:52:30.528218   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:30.530323  1897 raft_consensus.cc:491] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.530853  1897 raft_consensus.cc:513] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.533907  1897 leader_election.cc:290] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:30.535398   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:30.536834  1234 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.537271  1234 ts_tablet_manager.cc:1397] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.283s	user 0.014s	sys 0.006s
I20250628 07:52:30.539424  1234 raft_consensus.cc:357] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.540589  1234 raft_consensus.cc:383] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:30.539757  1062 leader_election.cc:343] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.541043  1234 raft_consensus.cc:738] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.541855  1234 consensus_queue.cc:260] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.544168  1234 ts_tablet_manager.cc:1428] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.007s	user 0.003s	sys 0.001s
W20250628 07:52:30.551185   758 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:30.546336  2014 tablet.cc:1722] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.575157  1234 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:30.579936   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_958abb2126a34f57af7942df2b6bbb9a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:30.581604   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_958abb2126a34f57af7942df2b6bbb9a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:30.590294   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c97313f05be54e4da2bc4cbbc04c8682 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:30.594576   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.385924   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9579dd44e5e241aea2671be28e16ef7c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:30.595438  1807 log.cc:1199] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/e28ab08f878d4d4b86a5646f687b08fc
W20250628 07:52:30.361734  1982 log.cc:927] Time spent T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Append to log took a long time: real 0.082s	user 0.000s	sys 0.000s
I20250628 07:52:30.437321   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 950fea8f6bfb409f8868fe5e88d3dddb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:30.596524   757 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.596927   757 leader_election.cc:304] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.457046   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:30.481922  1229 tablet_bootstrap.cc:654] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:30.598387  1062 leader_election.cc:343] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.598852  1062 leader_election.cc:304] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.599542  1897 raft_consensus.cc:2747] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.567027   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:30.591835   921 consensus_peers.cc:489] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:30.601337  1062 leader_election.cc:343] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.601788  1062 leader_election.cc:304] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.602459  1897 raft_consensus.cc:2747] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:30.604462   921 leader_election.cc:343] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:30.604966   921 leader_election.cc:304] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:30.605237  1230 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:30.605609  1230 ts_tablet_manager.cc:1397] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.179s	user 0.015s	sys 0.014s
I20250628 07:52:30.605597  1863 raft_consensus.cc:2747] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.607753  1230 raft_consensus.cc:357] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.608338  1230 raft_consensus.cc:383] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.608568  1230 raft_consensus.cc:738] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.596249  1807 ts_tablet_manager.cc:1939] T e28ab08f878d4d4b86a5646f687b08fc P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
W20250628 07:52:30.618294   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31], current schema version=3) failed: Not found: failed to reset TS proxy: No leader found: Tablet 661d92febf1142f99815f70ae9b27edc consensus state has no leader
I20250628 07:52:30.619516  1983 ts_tablet_manager.cc:1918] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:30.619935  1983 log.cc:1199] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:30.620525  1983 ts_tablet_manager.cc:1939] T b2d49e5a06384758bce5e7328b6a10f8 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:30.621148  2013 tablet.cc:1722] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.621847   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet e28ab08f878d4d4b86a5646f687b08fc (table default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]) successfully deleted
W20250628 07:52:30.623229   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because the tablet was not found. No further retry: Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:30.632840  1983 tablet_replica.cc:331] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:30.633208  1230 consensus_queue.cc:260] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.633817  1983 raft_consensus.cc:2241] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:30.634343  1983 raft_consensus.cc:2270] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.635411  1230 ts_tablet_manager.cc:1428] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.030s	user 0.005s	sys 0.000s
I20250628 07:52:30.600670  1987 raft_consensus.cc:2747] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.624030  1807 tablet_replica.cc:331] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:30.624804  1229 tablet_bootstrap.cc:492] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:30.645962  1229 ts_tablet_manager.cc:1397] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.397s	user 0.007s	sys 0.002s
I20250628 07:52:30.648888  1229 raft_consensus.cc:357] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.653424   986 raft_consensus.cc:3058] T 4ba7d2634cb74f6f858a5a3db8493b6a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:30.655086  1807 raft_consensus.cc:2241] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:30.675539  1234 tablet_bootstrap.cc:654] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.690996  1940 ts_tablet_manager.cc:1918] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:30.660317  1983 ts_tablet_manager.cc:1905] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.629119   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet b2d49e5a06384758bce5e7328b6a10f8 (table default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]) successfully deleted
W20250628 07:52:30.694533   533 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 65449.2us
I20250628 07:52:30.698783  1940 log.cc:1199] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/950fea8f6bfb409f8868fe5e88d3dddb
W20250628 07:52:30.703879   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 67430.6us
I20250628 07:52:30.712100  2014 tablet.cc:1722] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.714758  1234 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.715257  1234 ts_tablet_manager.cc:1397] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.140s	user 0.006s	sys 0.005s
I20250628 07:52:30.733878  1229 raft_consensus.cc:383] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.734934  1229 raft_consensus.cc:738] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.738107  1234 raft_consensus.cc:357] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.738888  1234 raft_consensus.cc:383] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.739189  1234 raft_consensus.cc:738] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.739791  1940 ts_tablet_manager.cc:1939] T 950fea8f6bfb409f8868fe5e88d3dddb P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:30.733666  2013 tablet.cc:1722] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.734606  1807 pending_rounds.cc:70] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
I20250628 07:52:30.739428  1229 consensus_queue.cc:260] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.745385  1234 consensus_queue.cc:260] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.741364  1807 pending_rounds.cc:77] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493921643212800 op_type: NO_OP noop_request { }
I20250628 07:52:30.747926  1807 raft_consensus.cc:2887] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:30.748302  1807 raft_consensus.cc:2270] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:30.752497  1230 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:30.742486   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 950fea8f6bfb409f8868fe5e88d3dddb (table default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]) successfully deleted
I20250628 07:52:30.770148  1234 ts_tablet_manager.cc:1428] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.055s	user 0.008s	sys 0.000s
I20250628 07:52:30.772993  1234 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:30.779063  1229 ts_tablet_manager.cc:1428] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.133s	user 0.005s	sys 0.000s
I20250628 07:52:30.787170   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_11e96390464141b9a308e3f956e1a84d" } modify_external_catalogs: true
I20250628 07:52:30.787725   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_11e96390464141b9a308e3f956e1a84d" } modify_external_catalogs: true
I20250628 07:52:30.790338   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b35ebcabf6704f049dc9352f2b2d212e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:30.793634  1230 tablet_bootstrap.cc:654] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.796789  1234 tablet_bootstrap.cc:654] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.806095  1229 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:30.795876  1807 ts_tablet_manager.cc:1905] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:30.822803  1897 raft_consensus.cc:491] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:30.823290  1897 raft_consensus.cc:513] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.831051  1897 leader_election.cc:290] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:30.832902   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:30.834300  1062 leader_election.cc:343] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:30.835786   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b35ebcabf6704f049dc9352f2b2d212e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:30.835860   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:30.836865  1062 leader_election.cc:343] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:30.837196  1062 leader_election.cc:304] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:30.839033  1234 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.839450  1234 ts_tablet_manager.cc:1397] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.067s	user 0.008s	sys 0.000s
W20250628 07:52:30.847606   531 consensus_peers.cc:489] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 -> Peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Couldn't send request to peer 5fd67221fa5f424ea0f83e582629d276. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20250628 07:52:30.847949  1229 tablet_bootstrap.cc:654] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.838393  1897 raft_consensus.cc:2747] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:30.848344  1234 raft_consensus.cc:357] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:30.852700  2022 log.cc:927] Time spent T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Append to log took a long time: real 0.065s	user 0.000s	sys 0.000s
I20250628 07:52:30.843811  2024 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.856395  1983 ts_tablet_manager.cc:1918] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:30.856822  1983 log.cc:1199] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/5ec8c1d9386a4922b55df5e06039858d
I20250628 07:52:30.857566  1983 ts_tablet_manager.cc:1939] T 5ec8c1d9386a4922b55df5e06039858d P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:30.848966  1234 raft_consensus.cc:383] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.861291  1234 raft_consensus.cc:738] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.861937  1234 consensus_queue.cc:260] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.864810  1234 ts_tablet_manager.cc:1428] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.025s	user 0.006s	sys 0.000s
I20250628 07:52:30.867772  2014 tablet.cc:1722] T 7eca25db66fa42a798fc002d43f7e3c3 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.878023  2036 tablet.cc:1722] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.885756  2029 tablet.cc:1722] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.925762  1234 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:30.930889  1234 tablet_bootstrap.cc:654] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.936556   801 tablet_service.cc:1468] Processing CreateTablet for tablet a168c27bfea94f9bba06c3a88329a892 (DEFAULT_TABLE table=default.Table_c974994f851542aa92291ccc82d531e0 [id=f7cf0e3d20f645d2ad5d9b5ee3637834]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.937513   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a168c27bfea94f9bba06c3a88329a892. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.940037   797 tablet_service.cc:1468] Processing CreateTablet for tablet 924a153449b64d41b7db8c46afed93a9 (DEFAULT_TABLE table=default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 [id=afffd13a151540f6863fc55f1a0c3f8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.941036   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924a153449b64d41b7db8c46afed93a9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.941358  1108 tablet_service.cc:1468] Processing CreateTablet for tablet a168c27bfea94f9bba06c3a88329a892 (DEFAULT_TABLE table=default.Table_c974994f851542aa92291ccc82d531e0 [id=f7cf0e3d20f645d2ad5d9b5ee3637834]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.947273  2036 tablet.cc:1722] T 7eca25db66fa42a798fc002d43f7e3c3 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:30.950292  1234 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:30.950768  1234 ts_tablet_manager.cc:1397] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.025s	user 0.010s	sys 0.001s
I20250628 07:52:30.964102   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 77234b63748849b6996c7aa0ca23ef05
I20250628 07:52:30.964164  1234 raft_consensus.cc:357] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.964792  1234 raft_consensus.cc:383] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:30.965035  1234 raft_consensus.cc:738] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:30.965615  1234 consensus_queue.cc:260] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.969915  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 77234b63748849b6996c7aa0ca23ef05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.971405   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 63115b7be4a94b639c18601d438ed947
I20250628 07:52:30.973484  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:30.974016   545 catalog_manager.cc:6652] ReplaceTablet: tablet 63115b7be4a94b639c18601d438ed947 deleted and replaced by tablet f0d48652a4da42f3b26b42f65c8db51b
I20250628 07:52:30.974395   545 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 330) took 1236 ms. Trace:
I20250628 07:52:30.979837  1234 ts_tablet_manager.cc:1428] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.000s
I20250628 07:52:30.980760  1234 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:30.982446   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 77234b63748849b6996c7aa0ca23ef05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:30.982460   799 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:30.985111   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 77234b63748849b6996c7aa0ca23ef05 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 77234b63748849b6996c7aa0ca23ef05 already in progress: creating tablet
W20250628 07:52:30.985911   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 63115b7be4a94b639c18601d438ed947 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 63115b7be4a94b639c18601d438ed947 already in progress: creating tablet
I20250628 07:52:30.986732  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a168c27bfea94f9bba06c3a88329a892. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.986843  1983 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 251) took 1005 ms. Trace:
I20250628 07:52:30.987200  1983 rpcz_store.cc:276] 0628 07:52:29.980973 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:30.019117 (+ 38144us) service_pool.cc:225] Handling call
0628 07:52:30.628763 (+609646us) ts_tablet_manager.cc:1145] Deleting tablet 5ec8c1d9386a4922b55df5e06039858d
0628 07:52:30.632701 (+  3938us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:30.776027 (+143326us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:30.856188 (+ 80161us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:30.986787 (+130599us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":601211}
I20250628 07:52:30.990598   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 5ec8c1d9386a4922b55df5e06039858d (table default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]) successfully deleted
I20250628 07:52:31.008360   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b0c733c64d5449ea86e9103a47c2f578" } new_table_name: "default.Table_cbbe18b888b7493a8c58ed399a96f972" modify_external_catalogs: true
I20250628 07:52:31.009841  1234 tablet_bootstrap.cc:654] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.012156  1230 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.012585  1230 ts_tablet_manager.cc:1397] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.260s	user 0.012s	sys 0.019s
I20250628 07:52:31.015141  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 924a153449b64d41b7db8c46afed93a9 (DEFAULT_TABLE table=default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 [id=afffd13a151540f6863fc55f1a0c3f8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.014978  1230 raft_consensus.cc:357] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:30.934152   956 tablet_service.cc:1468] Processing CreateTablet for tablet a168c27bfea94f9bba06c3a88329a892 (DEFAULT_TABLE table=default.Table_c974994f851542aa92291ccc82d531e0 [id=f7cf0e3d20f645d2ad5d9b5ee3637834]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:30.938962   961 tablet_service.cc:1468] Processing CreateTablet for tablet 924a153449b64d41b7db8c46afed93a9 (DEFAULT_TABLE table=default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 [id=afffd13a151540f6863fc55f1a0c3f8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.015829  1230 raft_consensus.cc:383] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.016084  1230 raft_consensus.cc:738] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.016430   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924a153449b64d41b7db8c46afed93a9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:30.979182   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 77234b63748849b6996c7aa0ca23ef05 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:30.979674   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:31.016680  1230 consensus_queue.cc:260] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.019487  1230 ts_tablet_manager.cc:1428] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.007s	user 0.003s	sys 0.004s
I20250628 07:52:31.019991   799 tablet_service.cc:1468] Processing CreateTablet for tablet fc4c7770fb564ccd95b3c789b179eecf (DEFAULT_TABLE table=default.Table_5f97f65bd045458e83a0d56856db2b89 [id=8c576a9b2b00448fac7e76983bde42e6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.020596   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a168c27bfea94f9bba06c3a88329a892. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.023061   955 tablet_service.cc:1468] Processing CreateTablet for tablet fc4c7770fb564ccd95b3c789b179eecf (DEFAULT_TABLE table=default.Table_5f97f65bd045458e83a0d56856db2b89 [id=8c576a9b2b00448fac7e76983bde42e6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.024044   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc4c7770fb564ccd95b3c789b179eecf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.020947   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc4c7770fb564ccd95b3c789b179eecf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.034946  1100 tablet_service.cc:1468] Processing CreateTablet for tablet fc4c7770fb564ccd95b3c789b179eecf (DEFAULT_TABLE table=default.Table_5f97f65bd045458e83a0d56856db2b89 [id=8c576a9b2b00448fac7e76983bde42e6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.043978  1983 tablet_replica.cc:331] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:31.044867  1983 raft_consensus.cc:2241] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:31.045383  1983 raft_consensus.cc:2270] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:31.047598   963 tablet_service.cc:1468] Processing CreateTablet for tablet 85c0ba070a8a40f2b32854ca24bc025c (DEFAULT_TABLE table=default.Table_04dc0144b3954979b9fdcce449c44782 [id=ad5a5ac2a4024b439c1d4fbe5e82509d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.048602   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85c0ba070a8a40f2b32854ca24bc025c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.054083  2014 tablet.cc:1722] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:31.055554  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc4c7770fb564ccd95b3c789b179eecf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.061514  2039 tablet.cc:1722] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:31.063884  1230 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:31.066231  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924a153449b64d41b7db8c46afed93a9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.071202  1230 tablet_bootstrap.cc:654] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:30.974730   545 rpcz_store.cc:276] 0628 07:52:29.737914 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.838494 (+100580us) service_pool.cc:225] Handling call
0628 07:52:29.901262 (+ 62768us) spinlock_profiling.cc:243] Waited 61.7 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea943891 00007f5be867c3a7 00007f5be867c332 00007f5be867c261 00007f5be867c1dc 00007f5be867b492 00007f5be6957dcc 00007f5be6957206
0628 07:52:30.974340 (+1073078us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:30.925262 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:30.925601 (+   339us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:30.925630 (+    29us) write_op.cc:435] Acquired schema lock
0628 07:52:30.925644 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:30.925954 (+   310us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:30.925989 (+    35us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:30.926009 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:30.926177 (+   168us) tablet.cc:699] Row locks acquired
0628 07:52:30.926189 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:30.926279 (+    90us) write_op.cc:270] Start()
0628 07:52:30.926347 (+    68us) write_op.cc:276] Timestamp: P: 1751097150926206 usec, L: 0
0628 07:52:30.926361 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:30.926772 (+   411us) log.cc:844] Serialized 628 byte log entry
0628 07:52:30.951273 (+ 24501us) op_driver.cc:464] REPLICATION: finished
0628 07:52:30.959699 (+  8426us) write_op.cc:301] APPLY: starting
0628 07:52:30.959747 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:30.959847 (+   100us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:30.959976 (+   129us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:30.960351 (+   375us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:30.960535 (+   184us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:30.960577 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:30.960735 (+   158us) log.cc:844] Serialized 29 byte log entry
0628 07:52:30.960854 (+   119us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:30.960920 (+    66us) write_op.cc:454] Released schema lock
0628 07:52:30.960947 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:30.961425 (+   478us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:30.961460 (+    35us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":141847296,"child_traces":[["op",{"apply.queue_time_us":8289,"apply.run_cpu_time_us":1450,"apply.run_wall_time_us":1667,"num_ops":2,"prepare.queue_time_us":7312,"prepare.run_cpu_time_us":1784,"prepare.run_wall_time_us":1785,"raft.queue_time_us":1457,"raft.run_cpu_time_us":465,"raft.run_wall_time_us":465,"replication_time_us":24824}]]}
I20250628 07:52:31.076448   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_be28b907172e4b36b346351fbc249f7a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:31.077589  2027 tablet.cc:1722] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
W20250628 07:52:31.078226   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_be28b907172e4b36b346351fbc249f7a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:31.078845  1229 tablet_bootstrap.cc:492] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:31.069854   542 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (table default.Table_df287bcea5bd414ebe5aee1689733f8c [id=398556bc3bdc481cb6577321cbc31055]). Expected version 1 got 0
I20250628 07:52:31.085476  1897 raft_consensus.cc:491] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.085950  1897 raft_consensus.cc:513] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.087679  1897 leader_election.cc:290] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:31.089274   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:31.090986  1062 leader_election.cc:343] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.091210   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet a447edf0960c4739930450a8c24ff6f7
I20250628 07:52:31.092343   547 catalog_manager.cc:6652] ReplaceTablet: tablet a447edf0960c4739930450a8c24ff6f7 deleted and replaced by tablet 924b921830b7403596ebcc6cf80dc24d
I20250628 07:52:31.094255   800 tablet_service.cc:1515] Processing DeleteTablet for tablet a447edf0960c4739930450a8c24ff6f7 with delete_type TABLET_DATA_DELETED (replaced by tablet 924b921830b7403596ebcc6cf80dc24d) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:31.095688   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet a447edf0960c4739930450a8c24ff6f7 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a447edf0960c4739930450a8c24ff6f7 already in progress: creating tablet
I20250628 07:52:31.096218  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet a447edf0960c4739930450a8c24ff6f7 with delete_type TABLET_DATA_DELETED (replaced by tablet 924b921830b7403596ebcc6cf80dc24d) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:31.096354   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:31.096326   955 tablet_service.cc:1515] Processing DeleteTablet for tablet a447edf0960c4739930450a8c24ff6f7 with delete_type TABLET_DATA_DELETED (replaced by tablet 924b921830b7403596ebcc6cf80dc24d) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:31.102119  1062 leader_election.cc:343] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.102478  1062 leader_election.cc:304] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.103394  1897 raft_consensus.cc:2747] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.105355  1229 ts_tablet_manager.cc:1397] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.299s	user 0.006s	sys 0.006s
I20250628 07:52:31.083243   542 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:31.110574  1983 ts_tablet_manager.cc:1905] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:31.114179  1229 raft_consensus.cc:357] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.118182  1229 raft_consensus.cc:383] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.125336  1229 raft_consensus.cc:738] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.129791  1229 consensus_queue.cc:260] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.112980   542 catalog_manager.cc:5582] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:31.139074  1229 ts_tablet_manager.cc:1428] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.033s	user 0.005s	sys 0.001s
I20250628 07:52:31.117535  1230 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.139112   801 tablet_service.cc:1468] Processing CreateTablet for tablet 85c0ba070a8a40f2b32854ca24bc025c (DEFAULT_TABLE table=default.Table_04dc0144b3954979b9fdcce449c44782 [id=ad5a5ac2a4024b439c1d4fbe5e82509d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.140097  1230 ts_tablet_manager.cc:1397] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.076s	user 0.017s	sys 0.000s
I20250628 07:52:31.142246  1230 raft_consensus.cc:357] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.142932  1230 raft_consensus.cc:383] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.143175  1230 raft_consensus.cc:738] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.143553  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 85c0ba070a8a40f2b32854ca24bc025c (DEFAULT_TABLE table=default.Table_04dc0144b3954979b9fdcce449c44782 [id=ad5a5ac2a4024b439c1d4fbe5e82509d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.143716  1230 consensus_queue.cc:260] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.144469  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85c0ba070a8a40f2b32854ca24bc025c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.145833  1230 ts_tablet_manager.cc:1428] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
I20250628 07:52:31.151649  1230 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:31.151834   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85c0ba070a8a40f2b32854ca24bc025c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.154484  1229 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:31.156440  1230 tablet_bootstrap.cc:654] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.148439   542 catalog_manager.cc:5582] T 36eebc4a46bd454da5bc176df2a9680f P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:31.163192  1807 ts_tablet_manager.cc:1918] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:31.174803  1807 log.cc:1199] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:31.177526  1807 ts_tablet_manager.cc:1939] T 0f29c8ffb7df410295278eee9c578b03 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:31.188620   956 tablet_service.cc:1468] Processing CreateTablet for tablet 878b25ab3282423fab41014a8ea8949c (DEFAULT_TABLE table=default.Table_d9f1fae1f46f45efbde92d35775fdd0e [id=e32e5fdf4ddb42d794ebe9d70ad300fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.189796   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 878b25ab3282423fab41014a8ea8949c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.190949  1230 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.191303  1230 ts_tablet_manager.cc:1397] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.045s	user 0.012s	sys 0.000s
I20250628 07:52:31.193267  1230 raft_consensus.cc:357] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.194149  1230 raft_consensus.cc:383] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.194392  1230 raft_consensus.cc:738] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.194582  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 268) took 1291 ms. Trace:
I20250628 07:52:31.194895  1807 rpcz_store.cc:276] 0628 07:52:29.902966 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.909183 (+  6217us) service_pool.cc:225] Handling call
0628 07:52:30.623887 (+714704us) ts_tablet_manager.cc:1145] Deleting tablet 0f29c8ffb7df410295278eee9c578b03
0628 07:52:30.623924 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:30.918212 (+294288us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:31.162972 (+244760us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:31.194527 (+ 31555us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":3208,"tablet-delete.queue_time_us":714004}
I20250628 07:52:31.195148  1230 consensus_queue.cc:260] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.196126   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 0f29c8ffb7df410295278eee9c578b03 (table default.Table_5f59babfe3234a8e83bfab516b219032 [id=4aa514190a934ea1851b50ad9642ccaf]) successfully deleted
I20250628 07:52:31.199599  1230 ts_tablet_manager.cc:1428] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.008s	user 0.007s	sys 0.000s
I20250628 07:52:31.172040  1229 tablet_bootstrap.cc:654] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.203964  1234 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:31.204399  1234 ts_tablet_manager.cc:1397] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.224s	user 0.022s	sys 0.006s
I20250628 07:52:31.207228  1234 raft_consensus.cc:357] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.208699   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_123910cc7356492f93d0e1d1df124ad9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:31.210274   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_123910cc7356492f93d0e1d1df124ad9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:31.210649  1229 tablet_bootstrap.cc:492] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:31.210990  1229 ts_tablet_manager.cc:1397] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.057s	user 0.007s	sys 0.001s
I20250628 07:52:31.215827  1897 raft_consensus.cc:491] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.216253  1897 raft_consensus.cc:513] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.213358  1229 raft_consensus.cc:357] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.218055  1897 leader_election.cc:290] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:31.218359  1229 raft_consensus.cc:383] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.219293  1230 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:31.224486  1230 tablet_bootstrap.cc:654] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.228091  1229 raft_consensus.cc:738] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.228363   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fa3ec14241e344029df3e39473181078
I20250628 07:52:31.233310  1229 consensus_queue.cc:260] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.233824  1807 tablet_replica.cc:331] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:31.220664   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:31.238152  1229 ts_tablet_manager.cc:1428] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.027s	user 0.006s	sys 0.000s
I20250628 07:52:31.238989  1229 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:31.241605  1807 raft_consensus.cc:2241] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:31.242046  1807 raft_consensus.cc:2270] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:31.243800  1807 ts_tablet_manager.cc:1905] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:31.245321   956 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:31.247256  1062 leader_election.cc:343] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.247833  1866 raft_consensus.cc:491] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.248409  1866 raft_consensus.cc:513] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.251894  1866 leader_election.cc:290] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:31.252597   920 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:31.258013   921 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:31.258342   921 leader_election.cc:304] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:31.258675  1983 ts_tablet_manager.cc:1918] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:31.259038  1983 log.cc:1199] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/950fea8f6bfb409f8868fe5e88d3dddb
I20250628 07:52:31.259616  1863 raft_consensus.cc:2747] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.259728  1983 ts_tablet_manager.cc:1939] T 950fea8f6bfb409f8868fe5e88d3dddb P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:31.264760  1229 tablet_bootstrap.cc:654] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.265506   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 950fea8f6bfb409f8868fe5e88d3dddb (table default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]) successfully deleted
W20250628 07:52:31.267729   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 77234b63748849b6996c7aa0ca23ef05 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 77234b63748849b6996c7aa0ca23ef05 already in progress: creating tablet
W20250628 07:52:31.268323   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 63115b7be4a94b639c18601d438ed947 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 63115b7be4a94b639c18601d438ed947 already in progress: creating tablet
W20250628 07:52:31.268872   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet a447edf0960c4739930450a8c24ff6f7 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a447edf0960c4739930450a8c24ff6f7 already in progress: creating tablet
I20250628 07:52:31.286890  1229 tablet_bootstrap.cc:492] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:31.287253  1229 ts_tablet_manager.cc:1397] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.048s	user 0.008s	sys 0.000s
I20250628 07:52:31.300729  1229 raft_consensus.cc:357] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.301388  1229 raft_consensus.cc:383] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.301656  1229 raft_consensus.cc:738] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.305904  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 878b25ab3282423fab41014a8ea8949c (DEFAULT_TABLE table=default.Table_d9f1fae1f46f45efbde92d35775fdd0e [id=e32e5fdf4ddb42d794ebe9d70ad300fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.307030  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 878b25ab3282423fab41014a8ea8949c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.220021   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:31.244699  1997 consensus_queue.cc:1035] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:31.282889  2025 raft_consensus.cc:491] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.301672   801 tablet_service.cc:1468] Processing CreateTablet for tablet 878b25ab3282423fab41014a8ea8949c (DEFAULT_TABLE table=default.Table_d9f1fae1f46f45efbde92d35775fdd0e [id=e32e5fdf4ddb42d794ebe9d70ad300fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.321759   956 tablet_service.cc:1468] Processing CreateTablet for tablet f0c4343648e144c9922d9e6c484abb71 (DEFAULT_TABLE table=default.Table_d8ead90f917d46b9beef6badf9823be1 [id=e658c2419bd748e98d19b83528587ca7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.322268  2025 raft_consensus.cc:513] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.322753   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 878b25ab3282423fab41014a8ea8949c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.322782   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c4343648e144c9922d9e6c484abb71. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.324049  2025 leader_election.cc:290] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:31.310570   758 consensus_peers.cc:489] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5ec8c1d9386a4922b55df5e06039858d. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:31.326383  1062 leader_election.cc:343] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.326606  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:31.326773  1062 leader_election.cc:304] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.327206  1122 raft_consensus.cc:2466] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:31.327776  1897 raft_consensus.cc:2747] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.328496   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:31.329746   757 leader_election.cc:343] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.330928  1234 raft_consensus.cc:383] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.330801   758 leader_election.cc:304] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.334372  2025 raft_consensus.cc:2802] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:31.334839  2025 raft_consensus.cc:491] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:31.335237  2025 raft_consensus.cc:3058] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.336228  1229 consensus_queue.cc:260] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.347386  2025 raft_consensus.cc:513] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.350198  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:31.350756  1122 raft_consensus.cc:3058] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.352964   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:31.354593   757 leader_election.cc:343] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.357949  1122 raft_consensus.cc:2466] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:31.359040   758 leader_election.cc:304] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.359531  1229 ts_tablet_manager.cc:1428] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.072s	user 0.007s	sys 0.000s
I20250628 07:52:31.359711  1997 raft_consensus.cc:2802] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:31.360160  1997 raft_consensus.cc:695] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:31.361147  1997 consensus_queue.cc:237] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.320070   799 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:31.333316  1234 raft_consensus.cc:738] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.367247  1234 consensus_queue.cc:260] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:31.367899   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fa3ec14241e344029df3e39473181078 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa3ec14241e344029df3e39473181078 already in progress: creating tablet
I20250628 07:52:31.369264  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:31.371347  1234 ts_tablet_manager.cc:1428] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.167s	user 0.001s	sys 0.005s
W20250628 07:52:31.371842   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fa3ec14241e344029df3e39473181078 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa3ec14241e344029df3e39473181078 already in progress: creating tablet
I20250628 07:52:31.384855   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d3053995fff94245ae716d9ea4118068"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:31.386278   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d3053995fff94245ae716d9ea4118068 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:31.392052  1897 raft_consensus.cc:491] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.392460  1897 raft_consensus.cc:513] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:31.393594   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet f9a553c69c254fdcaf413b53018bbfb2 consensus state has no leader
I20250628 07:52:31.394088  1897 leader_election.cc:290] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:31.394430   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7437789ae57b4f029d85e0722aa8fa84"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:31.395745   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7437789ae57b4f029d85e0722aa8fa84 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:31.398769  1928 raft_consensus.cc:491] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.399204  1928 raft_consensus.cc:513] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.400259   801 tablet_service.cc:1468] Processing CreateTablet for tablet f0c4343648e144c9922d9e6c484abb71 (DEFAULT_TABLE table=default.Table_d8ead90f917d46b9beef6badf9823be1 [id=e658c2419bd748e98d19b83528587ca7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.401219   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c4343648e144c9922d9e6c484abb71. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.402223   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b2d49e5a06384758bce5e7328b6a10f8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:31.403379   757 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.404055   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:31.405232   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:31.400897  1928 leader_election.cc:290] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.405777  2025 leader_election.cc:290] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:31.409400  1062 leader_election.cc:343] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:31.410385  1062 leader_election.cc:343] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.410825  1062 leader_election.cc:304] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.411477  1897 raft_consensus.cc:2747] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.414395   986 raft_consensus.cc:3058] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:31.423056   758 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:31.423398   758 leader_election.cc:304] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.424221  1997 consensus_queue.cc:1035] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:31.425894  1997 raft_consensus.cc:2747] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.429252   963 tablet_service.cc:1468] Processing CreateTablet for tablet e4bf0710bcf145a382516afd4ee09f0b (DEFAULT_TABLE table=default.Table_d65bc763056e4ce181baccde39c92ab9 [id=c3789b7fe8064911bd4095ae69bfd8e5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.430202   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4bf0710bcf145a382516afd4ee09f0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.304909  1807 ts_tablet_manager.cc:1918] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:31.431193   799 tablet_service.cc:1468] Processing CreateTablet for tablet e4bf0710bcf145a382516afd4ee09f0b (DEFAULT_TABLE table=default.Table_d65bc763056e4ce181baccde39c92ab9 [id=c3789b7fe8064911bd4095ae69bfd8e5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.432109   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4bf0710bcf145a382516afd4ee09f0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.433383  1897 raft_consensus.cc:491] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.433733  1897 raft_consensus.cc:513] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.435247  1897 leader_election.cc:290] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:31.433471   961 tablet_service.cc:1468] Processing CreateTablet for tablet 19e7760b95b94336a985cdd128cef8de (DEFAULT_TABLE table=default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 [id=9ed7f5db10a4467ea2551b27fd9a02d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.436133   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 19e7760b95b94336a985cdd128cef8de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.438426   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:31.440179  1062 leader_election.cc:343] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.445453   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:31.450649  1807 log.cc:1199] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/950fea8f6bfb409f8868fe5e88d3dddb
I20250628 07:52:31.451452  1807 ts_tablet_manager.cc:1939] T 950fea8f6bfb409f8868fe5e88d3dddb P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
W20250628 07:52:31.451416  1062 leader_election.cc:343] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.451797  1062 leader_election.cc:304] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.452821  1897 raft_consensus.cc:2747] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.454222   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:31.454926   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:31.456149   538 catalog_manager.cc:5582] T 4ba7d2634cb74f6f858a5a3db8493b6a P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:31.459933  1100 tablet_service.cc:1468] Processing CreateTablet for tablet e4bf0710bcf145a382516afd4ee09f0b (DEFAULT_TABLE table=default.Table_d65bc763056e4ce181baccde39c92ab9 [id=c3789b7fe8064911bd4095ae69bfd8e5]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.461027  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4bf0710bcf145a382516afd4ee09f0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.472640  1807 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 271) took 1035 ms. Trace:
I20250628 07:52:31.478394   538 catalog_manager.cc:5582] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:31.479729   538 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:31.483388  1230 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.483747  1230 ts_tablet_manager.cc:1397] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.265s	user 0.016s	sys 0.015s
W20250628 07:52:31.485230   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 66016.1us
I20250628 07:52:31.488721  1230 raft_consensus.cc:357] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.489347  1230 raft_consensus.cc:383] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.489603  1230 raft_consensus.cc:738] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.490728  1807 rpcz_store.cc:276] 0628 07:52:30.437020 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:30.437169 (+   149us) service_pool.cc:225] Handling call
0628 07:52:31.233673 (+796504us) ts_tablet_manager.cc:1145] Deleting tablet 950fea8f6bfb409f8868fe5e88d3dddb
0628 07:52:31.233717 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:31.270633 (+ 36916us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:31.304700 (+ 34067us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:31.472586 (+167886us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":7025,"tablet-delete.queue_time_us":599363}
I20250628 07:52:31.491780  1230 consensus_queue.cc:260] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.492868   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 950fea8f6bfb409f8868fe5e88d3dddb (table default.Table_9045e51711844332acd5d748d0bbc3af [id=894e6693438e4730a1fc97a7276cbb49]) successfully deleted
W20250628 07:52:31.495064   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 63115b7be4a94b639c18601d438ed947 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 63115b7be4a94b639c18601d438ed947 already in progress: creating tablet
W20250628 07:52:31.506482   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 77234b63748849b6996c7aa0ca23ef05 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 77234b63748849b6996c7aa0ca23ef05 already in progress: creating tablet
I20250628 07:52:31.507854  1230 ts_tablet_manager.cc:1428] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.023s	user 0.005s	sys 0.000s
W20250628 07:52:31.508926   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet a447edf0960c4739930450a8c24ff6f7 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a447edf0960c4739930450a8c24ff6f7 already in progress: creating tablet
W20250628 07:52:31.513060   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fa3ec14241e344029df3e39473181078 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fa3ec14241e344029df3e39473181078 already in progress: creating tablet
I20250628 07:52:31.516881  1108 tablet_service.cc:1468] Processing CreateTablet for tablet f0c4343648e144c9922d9e6c484abb71 (DEFAULT_TABLE table=default.Table_d8ead90f917d46b9beef6badf9823be1 [id=e658c2419bd748e98d19b83528587ca7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.518150  1230 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:31.519690  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c4343648e144c9922d9e6c484abb71. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.527391  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 19e7760b95b94336a985cdd128cef8de (DEFAULT_TABLE table=default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 [id=9ed7f5db10a4467ea2551b27fd9a02d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.528297  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 19e7760b95b94336a985cdd128cef8de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.529261  1234 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:31.530354   956 tablet_service.cc:1468] Processing CreateTablet for tablet d3501090b0a34ca9882dde55b19e3621 (DEFAULT_TABLE table=default.Table_4eb55153d17a4680b1de4c5ecba431d2 [id=8b90a6a3de494e0dbd4321b2c24291cc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.530344   797 tablet_service.cc:1468] Processing CreateTablet for tablet 81d9066872654fceb44b7fb63b60f070 (DEFAULT_TABLE table=default.Table_4ed2c883318848a19e2dac4d9820b2e3 [id=48ef616b338f4d7b932257bfb1b9f0be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.531394   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d9066872654fceb44b7fb63b60f070. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.531419   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3501090b0a34ca9882dde55b19e3621. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.533694  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 81d9066872654fceb44b7fb63b60f070 (DEFAULT_TABLE table=default.Table_4ed2c883318848a19e2dac4d9820b2e3 [id=48ef616b338f4d7b932257bfb1b9f0be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.534543  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d9066872654fceb44b7fb63b60f070. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.542253  1229 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:31.542807   757 consensus_peers.cc:489] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:31.555291   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a8b4bbb284774df48f62eda3df2ac372" } modify_external_catalogs: true
I20250628 07:52:31.555814   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a8b4bbb284774df48f62eda3df2ac372" } modify_external_catalogs: true
I20250628 07:52:31.559463   545 master_service.cc:948] ReplaceTablet: received request to replace tablet 532ba24d0d644313b9ef4e3ec10b0fed from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:31.564286   801 tablet_service.cc:1468] Processing CreateTablet for tablet 19e7760b95b94336a985cdd128cef8de (DEFAULT_TABLE table=default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 [id=9ed7f5db10a4467ea2551b27fd9a02d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.565778   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 19e7760b95b94336a985cdd128cef8de. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.569983   963 tablet_service.cc:1468] Processing CreateTablet for tablet 81d9066872654fceb44b7fb63b60f070 (DEFAULT_TABLE table=default.Table_4ed2c883318848a19e2dac4d9820b2e3 [id=48ef616b338f4d7b932257bfb1b9f0be]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.571399   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d9066872654fceb44b7fb63b60f070. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.577337  1230 tablet_bootstrap.cc:654] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.579051   544 master_service.cc:948] ReplaceTablet: received request to replace tablet 46ff641001cd4b01a4c09c0d8b79ebf3 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:31.585533  1997 raft_consensus.cc:491] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.586030  1997 raft_consensus.cc:513] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.589269  1997 leader_election.cc:290] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.590082  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:31.590757  1126 raft_consensus.cc:2466] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:31.591794   758 leader_election.cc:304] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:31.592782  1997 raft_consensus.cc:2802] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:31.593168  1997 raft_consensus.cc:491] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:31.593489  1997 raft_consensus.cc:3058] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.597574   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:31.598639   757 leader_election.cc:343] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.601205  2025 raft_consensus.cc:491] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.605664   800 tablet_service.cc:1468] Processing CreateTablet for tablet 5febe562643048f083e579c5f7ba0793 (DEFAULT_TABLE table=default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.606658   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5febe562643048f083e579c5f7ba0793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.606628   961 tablet_service.cc:1468] Processing CreateTablet for tablet 5febe562643048f083e579c5f7ba0793 (DEFAULT_TABLE table=default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.606663  1100 tablet_service.cc:1468] Processing CreateTablet for tablet d3501090b0a34ca9882dde55b19e3621 (DEFAULT_TABLE table=default.Table_4eb55153d17a4680b1de4c5ecba431d2 [id=8b90a6a3de494e0dbd4321b2c24291cc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.607498   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5febe562643048f083e579c5f7ba0793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.608158  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3501090b0a34ca9882dde55b19e3621. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.609812   955 tablet_service.cc:1468] Processing CreateTablet for tablet c3239c3041c84dfdbc61de4fcb4ea5dd (DEFAULT_TABLE table=default.Table_6541827cef9a442685f4b25c2539eb33 [id=38e154d3c3fc455eb314ff2d8323bb5b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.610886   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3239c3041c84dfdbc61de4fcb4ea5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.613720   798 tablet_service.cc:1468] Processing CreateTablet for tablet c3239c3041c84dfdbc61de4fcb4ea5dd (DEFAULT_TABLE table=default.Table_6541827cef9a442685f4b25c2539eb33 [id=38e154d3c3fc455eb314ff2d8323bb5b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.614562   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3239c3041c84dfdbc61de4fcb4ea5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.618377  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 5febe562643048f083e579c5f7ba0793 (DEFAULT_TABLE table=default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.619266  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5febe562643048f083e579c5f7ba0793. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.621452   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b35ebcabf6704f049dc9352f2b2d212e" } modify_external_catalogs: true
I20250628 07:52:31.621917   799 tablet_service.cc:1468] Processing CreateTablet for tablet d3501090b0a34ca9882dde55b19e3621 (DEFAULT_TABLE table=default.Table_4eb55153d17a4680b1de4c5ecba431d2 [id=8b90a6a3de494e0dbd4321b2c24291cc]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.622123   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b35ebcabf6704f049dc9352f2b2d212e" } modify_external_catalogs: true
I20250628 07:52:31.622810   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3501090b0a34ca9882dde55b19e3621. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.626147  1234 tablet_bootstrap.cc:654] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.601604  2025 raft_consensus.cc:513] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.638063  2025 leader_election.cc:290] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.639115  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:31.639725  1126 raft_consensus.cc:2466] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:31.640646   758 leader_election.cc:304] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:31.641418  2025 raft_consensus.cc:2802] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:31.641736  2025 raft_consensus.cc:491] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:31.642072  2025 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.663144  1230 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.663537  1230 ts_tablet_manager.cc:1397] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.146s	user 0.000s	sys 0.008s
I20250628 07:52:31.665695  1230 raft_consensus.cc:357] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.666270  1230 raft_consensus.cc:383] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.666515  1230 raft_consensus.cc:738] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.667321  1230 consensus_queue.cc:260] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.672237  1384 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:52:31.674270  1384 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:52:31.674786  1384 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:52:31.675582   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:31.679657   757 leader_election.cc:343] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.680111  2057 raft_consensus.cc:491] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.680454  2057 raft_consensus.cc:513] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.682163  2057 leader_election.cc:290] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.687026   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:31.691618   757 leader_election.cc:343] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.693218  1229 tablet_bootstrap.cc:654] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.698727  1109 tablet_service.cc:1468] Processing CreateTablet for tablet c3239c3041c84dfdbc61de4fcb4ea5dd (DEFAULT_TABLE table=default.Table_6541827cef9a442685f4b25c2539eb33 [id=38e154d3c3fc455eb314ff2d8323bb5b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.700093  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3239c3041c84dfdbc61de4fcb4ea5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.701372  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:31.701887  1126 raft_consensus.cc:2466] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:31.702862   758 leader_election.cc:304] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.703953  2057 raft_consensus.cc:2802] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:31.704217  2057 raft_consensus.cc:491] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:31.704466  2057 raft_consensus.cc:3058] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.710875  2057 raft_consensus.cc:513] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:31.729585   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet f9a553c69c254fdcaf413b53018bbfb2 consensus state has no leader
I20250628 07:52:31.735567  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 2595308cbc0d44f0abfd0a126f16d4df (DEFAULT_TABLE table=default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.737771   963 tablet_service.cc:1468] Processing CreateTablet for tablet 8d86b6da48254a608346474cf33be345 (DEFAULT_TABLE table=default.Table_df7fc4695669432f9b2bcc300360c70b [id=065317d32ef7414aa62609e873462803]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.739113   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d86b6da48254a608346474cf33be345. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.736950  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2595308cbc0d44f0abfd0a126f16d4df. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.741918   956 tablet_service.cc:1468] Processing CreateTablet for tablet 2595308cbc0d44f0abfd0a126f16d4df (DEFAULT_TABLE table=default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.742827   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2595308cbc0d44f0abfd0a126f16d4df. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.746083  1230 ts_tablet_manager.cc:1428] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.082s	user 0.005s	sys 0.000s
I20250628 07:52:31.758194   547 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 5177797ec399414ca840a05147af627d
I20250628 07:52:31.752715   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:31.760470   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:31.763396  1230 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:31.773399  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 8d86b6da48254a608346474cf33be345 (DEFAULT_TABLE table=default.Table_df7fc4695669432f9b2bcc300360c70b [id=065317d32ef7414aa62609e873462803]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.773937   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:31.774338  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d86b6da48254a608346474cf33be345. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.774338   961 tablet_service.cc:1468] Processing CreateTablet for tablet abf8f2d8069445bab27852e4e16a6bee (DEFAULT_TABLE table=default.Table_e10aee64839f49a18fc40406bb808709 [id=d607ae35fec24cab9d45e70c9ec351f3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.775291   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet abf8f2d8069445bab27852e4e16a6bee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.777733  1106 tablet_service.cc:1468] Processing CreateTablet for tablet abf8f2d8069445bab27852e4e16a6bee (DEFAULT_TABLE table=default.Table_e10aee64839f49a18fc40406bb808709 [id=d607ae35fec24cab9d45e70c9ec351f3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.778645  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet abf8f2d8069445bab27852e4e16a6bee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.782886   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:31.783087   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_d81430dc148d442e8dbfc9029c7cc3c1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 1 got 0
I20250628 07:52:31.783780   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
W20250628 07:52:31.784170   757 leader_election.cc:343] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.786192  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:31.786705  1126 raft_consensus.cc:3058] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.632923  1997 raft_consensus.cc:513] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.791028   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:31.792191   757 leader_election.cc:343] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.794912  1230 tablet_bootstrap.cc:654] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:31.795078  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:31.795542  1122 raft_consensus.cc:3058] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.806226  1122 raft_consensus.cc:2466] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:31.807233   758 leader_election.cc:304] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.807905  1928 raft_consensus.cc:2802] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:31.667444  2025 raft_consensus.cc:513] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.757535  1234 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:31.813542  1234 ts_tablet_manager.cc:1397] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.351s	user 0.007s	sys 0.002s
I20250628 07:52:31.815266  1997 leader_election.cc:290] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.817023   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:31.827279  1230 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:31.827724  1230 ts_tablet_manager.cc:1397] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.065s	user 0.009s	sys 0.000s
I20250628 07:52:31.830981  1230 raft_consensus.cc:357] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.831601  1230 raft_consensus.cc:383] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.831880  1230 raft_consensus.cc:738] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.771989   801 tablet_service.cc:1468] Processing CreateTablet for tablet 8d86b6da48254a608346474cf33be345 (DEFAULT_TABLE table=default.Table_df7fc4695669432f9b2bcc300360c70b [id=065317d32ef7414aa62609e873462803]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.775089   800 tablet_service.cc:1468] Processing CreateTablet for tablet abf8f2d8069445bab27852e4e16a6bee (DEFAULT_TABLE table=default.Table_e10aee64839f49a18fc40406bb808709 [id=d607ae35fec24cab9d45e70c9ec351f3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.837011   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet abf8f2d8069445bab27852e4e16a6bee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.777081   797 tablet_service.cc:1468] Processing CreateTablet for tablet 7986ee140247478eafc2d95aaec8f8b4 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.815791  1234 raft_consensus.cc:357] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.841267  1234 raft_consensus.cc:383] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.841511  1234 raft_consensus.cc:738] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.842026   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7986ee140247478eafc2d95aaec8f8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.842082  1234 consensus_queue.cc:260] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.769169   798 tablet_service.cc:1468] Processing CreateTablet for tablet 2595308cbc0d44f0abfd0a126f16d4df (DEFAULT_TABLE table=default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.843600   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2595308cbc0d44f0abfd0a126f16d4df. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.845446   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d86b6da48254a608346474cf33be345. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:31.818270   757 leader_election.cc:343] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.849400  1863 raft_consensus.cc:491] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.851995  1897 raft_consensus.cc:491] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:31.849880  1863 raft_consensus.cc:513] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.852419  1897 raft_consensus.cc:513] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.854168  1897 leader_election.cc:290] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:31.855178  1863 leader_election.cc:290] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.856153   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:31.856153   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5475a89692eb450d82cde521ec8c9a73" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:31.856715   821 raft_consensus.cc:2466] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:31.857581   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:31.858355   921 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73
W20250628 07:52:31.858783  1062 leader_election.cc:343] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.858831   920 leader_election.cc:304] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
W20250628 07:52:31.859289  1062 leader_election.cc:343] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:31.859579  1062 leader_election.cc:304] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:31.859691  1866 raft_consensus.cc:2802] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:31.859987  1866 raft_consensus.cc:491] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:31.860464  1897 raft_consensus.cc:2747] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:31.860788  1866 raft_consensus.cc:3058] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.867205  1866 raft_consensus.cc:513] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.869903   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5475a89692eb450d82cde521ec8c9a73" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:31.870410   818 raft_consensus.cc:3058] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.870389  1866 leader_election.cc:290] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:31.871276   921 leader_election.cc:343] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:31.876698   818 raft_consensus.cc:2466] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:31.878734   920 leader_election.cc:304] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:31.879895  1863 raft_consensus.cc:2802] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:31.880555  1863 raft_consensus.cc:695] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:31.881415  1863 consensus_queue.cc:237] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.889374  1234 ts_tablet_manager.cc:1428] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.076s	user 0.007s	sys 0.006s
I20250628 07:52:31.897135  2025 leader_election.cc:290] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.909986  2057 leader_election.cc:290] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:31.897399  1229 tablet_bootstrap.cc:492] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:31.914178  1230 consensus_queue.cc:260] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.916209  1229 ts_tablet_manager.cc:1397] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.374s	user 0.011s	sys 0.000s
I20250628 07:52:31.921837  1928 raft_consensus.cc:695] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:31.922680  1928 consensus_queue.cc:237] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.947140   544 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 262328d78b904fa7a53c4350f923966f
I20250628 07:52:31.959126  1123 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:31.959708  1123 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:31.959949  1230 ts_tablet_manager.cc:1428] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.132s	user 0.003s	sys 0.001s
I20250628 07:52:31.969538   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_57d70b50d59c4f34bc68e39e92d7a6ef"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:31.971072   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_57d70b50d59c4f34bc68e39e92d7a6ef in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:31.989921  1123 raft_consensus.cc:2466] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:31.990360  1229 raft_consensus.cc:357] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:31.991005  1229 raft_consensus.cc:383] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:31.991251  1229 raft_consensus.cc:738] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:31.996620   955 tablet_service.cc:1468] Processing CreateTablet for tablet 7986ee140247478eafc2d95aaec8f8b4 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:31.997700   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7986ee140247478eafc2d95aaec8f8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:31.996416  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 7986ee140247478eafc2d95aaec8f8b4 (DEFAULT_TABLE table=default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:32.002377   758 leader_election.cc:304] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.003641  2025 raft_consensus.cc:2802] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.004046  2025 raft_consensus.cc:695] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:32.004760  2025 consensus_queue.cc:237] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.014639  1126 raft_consensus.cc:2466] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:32.024663   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 532ba24d0d644313b9ef4e3ec10b0fed
I20250628 07:52:32.016031  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7986ee140247478eafc2d95aaec8f8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:32.030326   545 catalog_manager.cc:6652] ReplaceTablet: tablet 532ba24d0d644313b9ef4e3ec10b0fed deleted and replaced by tablet c84abdf28d7b4c079d3e6daee9327030
I20250628 07:52:32.032526  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:32.035485   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:32.036373   799 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:32.017167  1230 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.045989   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_46b78ac794754d8da31f4b39fb3f8fca" } new_table_name: "default.Table_27b46453d69840dc832603c0d1ffea0f" modify_external_catalogs: true
I20250628 07:52:32.048641  1230 tablet_bootstrap.cc:654] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.054370   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]). Expected version 1 got 0
I20250628 07:52:32.055557  2072 tablet_replica.cc:331] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:32.055467   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]). Expected version 2 got 0
I20250628 07:52:32.056600  2072 raft_consensus.cc:2241] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:32.057546  2072 raft_consensus.cc:2270] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:32.057332   539 catalog_manager.cc:5582] T 132cd82d39e84e85a37f21ae208fd165 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:32.059715  2072 ts_tablet_manager.cc:1905] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:32.061931   758 leader_election.cc:304] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.062981  1928 raft_consensus.cc:2802] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.063395  1928 raft_consensus.cc:695] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:32.064144  1928 consensus_queue.cc:237] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.089545  1234 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.095472   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:32.099704  2071 tablet_replica.cc:331] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:32.101038  2071 raft_consensus.cc:2241] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:32.101214  1230 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.103312  1230 ts_tablet_manager.cc:1397] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.086s	user 0.009s	sys 0.000s
I20250628 07:52:32.109324  2073 tablet_replica.cc:331] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:32.110544  2073 raft_consensus.cc:2241] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:32.096838   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:32.114755  2073 raft_consensus.cc:2270] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:32.101953  2071 raft_consensus.cc:2270] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:31.991842  1229 consensus_queue.cc:260] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.115983  1897 raft_consensus.cc:491] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.105433  1230 raft_consensus.cc:357] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.117910  1230 raft_consensus.cc:383] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.117609  1897 raft_consensus.cc:513] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.118157  1230 raft_consensus.cc:738] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.119411  1897 leader_election.cc:290] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.121520   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.122699   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.122809  1062 leader_election.cc:343] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:32.124089  1062 leader_election.cc:343] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.124419  1062 leader_election.cc:304] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.124276  1230 consensus_queue.cc:260] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.125427  1897 raft_consensus.cc:2747] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.134514  1230 ts_tablet_manager.cc:1428] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.031s	user 0.005s	sys 0.000s
I20250628 07:52:32.135875  1863 raft_consensus.cc:491] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.136317  1863 raft_consensus.cc:513] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.138137  1863 leader_election.cc:290] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:32.140717   921 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:32.142027  1229 ts_tablet_manager.cc:1428] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.222s	user 0.005s	sys 0.000s
I20250628 07:52:32.141991   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9579dd44e5e241aea2671be28e16ef7c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.142510   824 raft_consensus.cc:2466] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:32.142928  1229 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.145157   920 leader_election.cc:304] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:32.146062  1863 raft_consensus.cc:2802] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:32.146391  1863 raft_consensus.cc:491] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.146709  1863 raft_consensus.cc:3058] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.147698  2073 ts_tablet_manager.cc:1905] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:32.148301  2071 ts_tablet_manager.cc:1905] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:32.154222  1863 raft_consensus.cc:513] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.156651   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9579dd44e5e241aea2671be28e16ef7c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:32.157251   821 raft_consensus.cc:3058] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.163108  1863 leader_election.cc:290] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.164278   821 raft_consensus.cc:2466] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:32.165433   920 leader_election.cc:304] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
W20250628 07:52:32.165923   921 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:32.166150  1863 raft_consensus.cc:2802] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.165881   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7ac89842d7e54a2ea03d7f64f4a92175"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:32.167366   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7ac89842d7e54a2ea03d7f64f4a92175 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:32.169797  1863 raft_consensus.cc:695] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:32.170563  1863 consensus_queue.cc:237] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.174544  1230 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:32.181406   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet fef70db0996f48c69574337f8f7ecf37 consensus state has no leader
I20250628 07:52:32.185658  2077 raft_consensus.cc:491] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.186560  1229 tablet_bootstrap.cc:654] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.186205  2077 raft_consensus.cc:513] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.189234   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:32.190689  1062 leader_election.cc:343] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.192075  2077 leader_election.cc:290] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.192729   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.194053  1062 leader_election.cc:343] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.194414  1062 leader_election.cc:304] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.195446  2077 raft_consensus.cc:2747] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.203588  2072 ts_tablet_manager.cc:1918] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:32.208070  2077 raft_consensus.cc:491] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.208618  2077 raft_consensus.cc:513] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.210719  1928 raft_consensus.cc:491] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.211274  1928 raft_consensus.cc:513] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.214421   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:32.206214  2072 log.cc:1199] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/532ba24d0d644313b9ef4e3ec10b0fed
W20250628 07:52:32.215627   757 leader_election.cc:343] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.214771  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.215834  2072 ts_tablet_manager.cc:1939] T 532ba24d0d644313b9ef4e3ec10b0fed P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:32.216238  1126 raft_consensus.cc:2466] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:32.217855   758 leader_election.cc:304] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.218693  2025 raft_consensus.cc:2802] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:32.219023  2025 raft_consensus.cc:491] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.219348  2025 raft_consensus.cc:3058] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.225188  1928 leader_election.cc:290] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.228494  2025 raft_consensus.cc:513] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.230922   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:32.231194  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:32.231680  1126 raft_consensus.cc:3058] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:32.232075   757 leader_election.cc:343] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.233947  1229 tablet_bootstrap.cc:492] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.234387  1229 ts_tablet_manager.cc:1397] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.092s	user 0.008s	sys 0.002s
I20250628 07:52:32.236135  1230 tablet_bootstrap.cc:654] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.237144  1229 raft_consensus.cc:357] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.241808   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]) successfully deleted
I20250628 07:52:32.242218  1229 raft_consensus.cc:383] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.242560  1229 raft_consensus.cc:738] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.243265  1229 consensus_queue.cc:260] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.248800  2025 leader_election.cc:290] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.255954  2077 leader_election.cc:290] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.257275   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.258029   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.258509  1062 leader_election.cc:343] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:32.259127  1062 leader_election.cc:343] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.259425  1062 leader_election.cc:304] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.264055  1229 ts_tablet_manager.cc:1428] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.029s	user 0.004s	sys 0.000s
I20250628 07:52:32.265143  1229 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.268922   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:32.270635   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:32.271637   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:32.272507  1126 raft_consensus.cc:2466] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:32.275831   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111), 
I20250628 07:52:32.276376   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:32.277832   758 leader_election.cc:304] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.278944  2025 raft_consensus.cc:2802] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.276786  2077 raft_consensus.cc:2747] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.279873   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:32.281908  2025 raft_consensus.cc:695] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:32.283201  2025 consensus_queue.cc:237] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.284791   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:32.285390   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:32.287067  1229 tablet_bootstrap.cc:654] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.290148  1234 tablet_bootstrap.cc:654] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.338847  1234 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.338847  1230 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.339241  1234 ts_tablet_manager.cc:1397] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.250s	user 0.014s	sys 0.000s
I20250628 07:52:32.339346  1230 ts_tablet_manager.cc:1397] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.165s	user 0.011s	sys 0.000s
I20250628 07:52:32.346030  1230 raft_consensus.cc:357] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:32.346422   531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111) [suppressed 5 similar messages]
I20250628 07:52:32.348318  1230 raft_consensus.cc:383] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.348575  1230 raft_consensus.cc:738] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.349918  1230 consensus_queue.cc:260] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.368132  1229 tablet_bootstrap.cc:492] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.367869  2094 tablet.cc:1722] T febd000d51e2480e81766b292304b4f5 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:32.368525  1229 ts_tablet_manager.cc:1397] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.104s	user 0.023s	sys 0.003s
I20250628 07:52:32.368604  2071 ts_tablet_manager.cc:1918] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:32.370749  1229 raft_consensus.cc:357] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.371333  1229 raft_consensus.cc:383] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.371573  1229 raft_consensus.cc:738] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.373842  1234 raft_consensus.cc:357] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.374574  1229 consensus_queue.cc:260] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.383519  2071 log.cc:1199] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/532ba24d0d644313b9ef4e3ec10b0fed
I20250628 07:52:32.384224  1229 ts_tablet_manager.cc:1428] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.015s	user 0.001s	sys 0.005s
I20250628 07:52:32.385576  2071 ts_tablet_manager.cc:1939] T 532ba24d0d644313b9ef4e3ec10b0fed P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:32.388772   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]) successfully deleted
I20250628 07:52:32.389648  1234 raft_consensus.cc:383] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.389976  1234 raft_consensus.cc:738] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.390710  1234 consensus_queue.cc:260] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.281991  2073 ts_tablet_manager.cc:1918] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:32.392304  1229 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.392351  2073 log.cc:1199] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/532ba24d0d644313b9ef4e3ec10b0fed
I20250628 07:52:32.393409  2073 ts_tablet_manager.cc:1939] T 532ba24d0d644313b9ef4e3ec10b0fed P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:32.396229   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 532ba24d0d644313b9ef4e3ec10b0fed (table default.Table_58cbee214d3d4a7ab7452f436eb56db3 [id=9c8759743f2c49cf911b12ca926da61f]) successfully deleted
I20250628 07:52:32.379643  2093 tablet.cc:1722] T febd000d51e2480e81766b292304b4f5 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:32.406482  1234 ts_tablet_manager.cc:1428] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.067s	user 0.005s	sys 0.001s
I20250628 07:52:32.408713  1234 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.417058  1229 tablet_bootstrap.cc:654] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.419950  1230 ts_tablet_manager.cc:1428] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.080s	user 0.006s	sys 0.000s
I20250628 07:52:32.422938  1234 tablet_bootstrap.cc:654] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:32.426846   757 consensus_peers.cc:489] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:32.428773  1928 consensus_queue.cc:1035] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:32.440023  1229 tablet_bootstrap.cc:492] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.440421  1229 ts_tablet_manager.cc:1397] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.055s	user 0.005s	sys 0.004s
I20250628 07:52:32.443250  1229 raft_consensus.cc:357] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.443845  1229 raft_consensus.cc:383] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.444072  1229 raft_consensus.cc:738] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.444924  1229 consensus_queue.cc:260] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.446112  1230 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.459854  1229 ts_tablet_manager.cc:1428] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.019s	user 0.006s	sys 0.000s
I20250628 07:52:32.462405  1230 tablet_bootstrap.cc:654] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.474316   986 raft_consensus.cc:3058] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.492071  1229 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.500303  1229 tablet_bootstrap.cc:654] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.514119  1230 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.514511  1230 ts_tablet_manager.cc:1397] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.069s	user 0.010s	sys 0.000s
I20250628 07:52:32.519017  1230 raft_consensus.cc:357] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.519688  1230 raft_consensus.cc:383] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.519961  1230 raft_consensus.cc:738] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.525345  1230 consensus_queue.cc:260] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.527734  1230 ts_tablet_manager.cc:1428] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:32.527468  2110 tablet.cc:1722] T 7eca25db66fa42a798fc002d43f7e3c3 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:32.528592  1230 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.530597  1229 tablet_bootstrap.cc:492] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.531023  1229 ts_tablet_manager.cc:1397] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.070s	user 0.008s	sys 0.000s
I20250628 07:52:32.532435  1234 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.537009  1234 ts_tablet_manager.cc:1397] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.128s	user 0.008s	sys 0.003s
I20250628 07:52:32.538478  1229 raft_consensus.cc:357] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.539090  1229 raft_consensus.cc:383] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.539320  1229 raft_consensus.cc:738] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.542043  1230 tablet_bootstrap.cc:654] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.544754  1229 consensus_queue.cc:260] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.545867  1234 raft_consensus.cc:357] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.550112  1234 raft_consensus.cc:383] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.550887  1234 raft_consensus.cc:738] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.554275  1863 raft_consensus.cc:491] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.554620  1229 ts_tablet_manager.cc:1428] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.023s	user 0.007s	sys 0.000s
I20250628 07:52:32.554703  1863 raft_consensus.cc:513] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.555569  1229 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.556519  1863 leader_election.cc:290] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.556802  1230 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.557196  1230 ts_tablet_manager.cc:1397] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.029s	user 0.009s	sys 0.002s
I20250628 07:52:32.558593  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.559382  1230 raft_consensus.cc:357] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.559849   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46e0aa090f544e8aa0a426c61925f345" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.560011  1230 raft_consensus.cc:383] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.560256  1230 raft_consensus.cc:738] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.561911   921 leader_election.cc:304] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:32.561518  1234 consensus_queue.cc:260] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.563339  1863 raft_consensus.cc:2747] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.564730  1230 consensus_queue.cc:260] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.567078  1230 ts_tablet_manager.cc:1428] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.010s	user 0.003s	sys 0.002s
I20250628 07:52:32.567914  1230 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.573483  1230 tablet_bootstrap.cc:654] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.578909  1229 tablet_bootstrap.cc:654] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.579602  1234 ts_tablet_manager.cc:1428] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.042s	user 0.005s	sys 0.001s
I20250628 07:52:32.587189  1234 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.607302  1229 tablet_bootstrap.cc:492] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.607725  1229 ts_tablet_manager.cc:1397] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.052s	user 0.009s	sys 0.000s
I20250628 07:52:32.610090  1229 raft_consensus.cc:357] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.610746  1229 raft_consensus.cc:383] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.610994  1229 raft_consensus.cc:738] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.613730  1229 consensus_queue.cc:260] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.617019  1234 tablet_bootstrap.cc:654] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.621891  1229 ts_tablet_manager.cc:1428] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.014s	user 0.005s	sys 0.002s
I20250628 07:52:32.626776  1229 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.627842  1230 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.628225  1230 ts_tablet_manager.cc:1397] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.060s	user 0.010s	sys 0.001s
I20250628 07:52:32.629065  2025 raft_consensus.cc:491] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.629520  2025 raft_consensus.cc:513] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.630486  1230 raft_consensus.cc:357] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.631129  1230 raft_consensus.cc:383] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.631377  1230 raft_consensus.cc:738] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.631372  2025 leader_election.cc:290] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.631928  1229 tablet_bootstrap.cc:654] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.632221  1230 consensus_queue.cc:260] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.637321  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.637940  1126 raft_consensus.cc:2466] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:32.639528   758 leader_election.cc:304] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:32.640799   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.641960   757 leader_election.cc:343] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.642561  1866 raft_consensus.cc:491] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.642984  1866 raft_consensus.cc:513] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.642710  2025 raft_consensus.cc:2802] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:32.644042  2025 raft_consensus.cc:491] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.644342  2025 raft_consensus.cc:3058] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.644804  1866 leader_election.cc:290] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.647104   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.648865  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "661d92febf1142f99815f70ae9b27edc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.650035  1126 raft_consensus.cc:2391] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
I20250628 07:52:32.639807  1230 ts_tablet_manager.cc:1428] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.001s
I20250628 07:52:32.651018   921 leader_election.cc:304] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:32.652364  1230 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.657265  1866 raft_consensus.cc:3058] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.660423  2025 raft_consensus.cc:513] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.663133   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:32.664386   757 leader_election.cc:343] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.664773  1230 tablet_bootstrap.cc:654] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.666230  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:32.666759  1126 raft_consensus.cc:3058] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.671154  1866 raft_consensus.cc:2747] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.676415  1229 tablet_bootstrap.cc:492] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.682492  1229 ts_tablet_manager.cc:1397] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.056s	user 0.010s	sys 0.000s
W20250628 07:52:32.686841   757 consensus_peers.cc:489] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:32.690809  1126 raft_consensus.cc:2466] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:32.691771   758 leader_election.cc:304] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.687193  1229 raft_consensus.cc:357] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.692512  1928 raft_consensus.cc:2802] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.692734  1229 raft_consensus.cc:383] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.692929  1928 raft_consensus.cc:695] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:32.692994  1229 raft_consensus.cc:738] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.694244  1928 consensus_queue.cc:237] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.697613  1229 consensus_queue.cc:260] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.703533  1229 ts_tablet_manager.cc:1428] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.021s	user 0.006s	sys 0.000s
I20250628 07:52:32.704430  1229 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.719921  2077 raft_consensus.cc:491] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.720654  1229 tablet_bootstrap.cc:654] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.720382  2077 raft_consensus.cc:513] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.722873  2077 leader_election.cc:290] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.724895   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.726615  1062 leader_election.cc:343] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.725188   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:32.731158  1062 leader_election.cc:343] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.731699  1062 leader_election.cc:304] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.732769  2077 raft_consensus.cc:2747] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.717824  2025 leader_election.cc:290] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.735939  1230 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.736327  1230 ts_tablet_manager.cc:1397] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.085s	user 0.017s	sys 0.004s
I20250628 07:52:32.738617  1230 raft_consensus.cc:357] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.739279  1230 raft_consensus.cc:383] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.739552  1230 raft_consensus.cc:738] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.740484  1230 consensus_queue.cc:260] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.757726  1230 ts_tablet_manager.cc:1428] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.021s	user 0.006s	sys 0.001s
I20250628 07:52:32.758018  1229 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.758595  1230 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.758741  1229 ts_tablet_manager.cc:1397] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.054s	user 0.010s	sys 0.000s
I20250628 07:52:32.760924  1229 raft_consensus.cc:357] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.762266  1229 raft_consensus.cc:383] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.762499  1229 raft_consensus.cc:738] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.763160  1229 consensus_queue.cc:260] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.766697  1229 ts_tablet_manager.cc:1428] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
I20250628 07:52:32.767505  1229 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:32.772027   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet fef70db0996f48c69574337f8f7ecf37 consensus state has no leader
I20250628 07:52:32.763098  1234 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.776383  1234 ts_tablet_manager.cc:1397] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.189s	user 0.011s	sys 0.000s
I20250628 07:52:32.777351  2117 raft_consensus.cc:491] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.777758  2117 raft_consensus.cc:513] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.779518  2117 leader_election.cc:290] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.781263  1230 tablet_bootstrap.cc:654] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.781646   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.783304  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "febd000d51e2480e81766b292304b4f5" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.784535   921 leader_election.cc:304] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:32.785418  2117 raft_consensus.cc:2747] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.790843  1234 raft_consensus.cc:357] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.792862  1230 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.793273  1230 ts_tablet_manager.cc:1397] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.035s	user 0.006s	sys 0.003s
I20250628 07:52:32.793417  1234 raft_consensus.cc:383] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.793776  1234 raft_consensus.cc:738] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.794864  1234 consensus_queue.cc:260] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.795464  1230 raft_consensus.cc:357] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.796049  1230 raft_consensus.cc:383] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.796291  1230 raft_consensus.cc:738] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.796898  1230 consensus_queue.cc:260] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.804020  1229 tablet_bootstrap.cc:654] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.813942  1230 ts_tablet_manager.cc:1428] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.020s	user 0.005s	sys 0.000s
I20250628 07:52:32.815778  1230 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:32.820861  1230 tablet_bootstrap.cc:654] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:32.824266   921 consensus_peers.cc:489] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:32.825289  2128 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:32.825802  2128 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:32.830384  1234 ts_tablet_manager.cc:1428] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.053s	user 0.005s	sys 0.000s
I20250628 07:52:32.831672  1229 tablet_bootstrap.cc:492] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.831668  2077 raft_consensus.cc:491] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.832087  1229 ts_tablet_manager.cc:1397] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.065s	user 0.006s	sys 0.002s
I20250628 07:52:32.832082  2077 raft_consensus.cc:513] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.833921  2077 leader_election.cc:290] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.834297  1229 raft_consensus.cc:357] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.834896  1229 raft_consensus.cc:383] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.835139  1229 raft_consensus.cc:738] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.835800   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:32.836850   983 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.842612   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:32.844595  1062 leader_election.cc:343] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.843755  1234 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:32.845561  1062 leader_election.cc:343] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.845327  1229 consensus_queue.cc:260] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.845877  1062 leader_election.cc:304] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.847036  2077 raft_consensus.cc:2747] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.851779  2117 raft_consensus.cc:491] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.852223  2117 raft_consensus.cc:513] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.854964  1234 tablet_bootstrap.cc:654] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.855376  2117 leader_election.cc:290] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.855998   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.856897  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.857462  1126 raft_consensus.cc:2466] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:32.858464  2025 consensus_queue.cc:1035] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:32.859338   921 leader_election.cc:304] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:32.860330  2117 raft_consensus.cc:2802] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:52:32.860661  2117 raft_consensus.cc:491] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.861045  2117 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:32.850747  1229 ts_tablet_manager.cc:1428] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.001s
I20250628 07:52:32.870654  1234 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.871042  1234 ts_tablet_manager.cc:1397] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.027s	user 0.012s	sys 0.000s
I20250628 07:52:32.872429  1229 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.875190  1234 raft_consensus.cc:357] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.875802  1234 raft_consensus.cc:383] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.876014  1234 raft_consensus.cc:738] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.875967   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 411 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:32.879146  2128 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:32.879122  1234 consensus_queue.cc:260] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.879479  2117 raft_consensus.cc:513] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.880721   983 raft_consensus.cc:1238] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Rejecting Update request from peer 6005c44dfe4848f18a304621cd49e934 for earlier term 1. Current term is 2. Ops: [1.1-1.1]
I20250628 07:52:32.882051   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:32.883881  1234 ts_tablet_manager.cc:1428] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20250628 07:52:32.885288  2025 consensus_queue.cc:1046] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: INVALID_TERM, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:32.886144  2025 raft_consensus.cc:3053] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Stepping down as leader of term 1
I20250628 07:52:32.886471  2025 raft_consensus.cc:738] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:32.887265  1234 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.887202  2025 consensus_queue.cc:260] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.888445  2077 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.888860  2077 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.890565  2077 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:32.892463   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:32.892740  2025 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Advancing to term 2
W20250628 07:52:32.893654  1062 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.893819  1229 tablet_bootstrap.cc:654] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.894475   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:32.895725  1062 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.896042  1062 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.897006  2077 raft_consensus.cc:2747] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:32.897260  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7e2ecd14e8524f018dd3c0aecd564218" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:32.888064  1230 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:32.897742  1126 raft_consensus.cc:3058] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:32.898108  1230 ts_tablet_manager.cc:1397] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.083s	user 0.009s	sys 0.000s
I20250628 07:52:32.898201  1234 tablet_bootstrap.cc:654] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:32.898808   610 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111)
I20250628 07:52:32.905012  2117 leader_election.cc:290] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.907203  1234 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.907534  1234 ts_tablet_manager.cc:1397] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.020s	user 0.007s	sys 0.002s
I20250628 07:52:32.912155  1234 raft_consensus.cc:357] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.914405  1234 raft_consensus.cc:383] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.904706  1126 raft_consensus.cc:2466] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 2.
I20250628 07:52:32.915746  1234 raft_consensus.cc:738] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.916702  1234 consensus_queue.cc:260] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.917820   921 leader_election.cc:304] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:32.919162  1863 raft_consensus.cc:2802] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Leader election won for term 2
I20250628 07:52:32.920476  1230 raft_consensus.cc:357] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.921264  1234 ts_tablet_manager.cc:1428] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:32.925222  1863 raft_consensus.cc:695] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:32.925980  1234 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.926013  1863 consensus_queue.cc:237] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:32.935775   610 leader_election.cc:336] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209): Network error: Client connection negotiation failed: client connection to 127.0.107.190:37209: connect: Connection refused (error 111)
I20250628 07:52:32.936195   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:32.936945  2128 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:32.938490  2025 raft_consensus.cc:491] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.936650  1234 tablet_bootstrap.cc:654] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.938936  2025 raft_consensus.cc:513] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.939813  1230 raft_consensus.cc:383] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.940754  2025 leader_election.cc:290] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.941404  1230 raft_consensus.cc:738] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.942025   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:32.941653  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.943471  1126 raft_consensus.cc:2466] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:32.943401   757 leader_election.cc:343] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.944427   758 leader_election.cc:304] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:32.945326  2025 raft_consensus.cc:2802] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:32.945688  2025 raft_consensus.cc:491] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.945976  2025 raft_consensus.cc:3058] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.949823  1229 tablet_bootstrap.cc:492] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:32.951005  1863 raft_consensus.cc:491] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:32.953313  1863 raft_consensus.cc:513] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.955638  1863 leader_election.cc:290] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.956321  1229 ts_tablet_manager.cc:1397] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.084s	user 0.010s	sys 0.000s
I20250628 07:52:32.956835  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:32.957559  1122 raft_consensus.cc:2466] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:32.958837   921 leader_election.cc:304] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:32.958931  1229 raft_consensus.cc:357] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.959713  1229 raft_consensus.cc:383] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.960013  1229 raft_consensus.cc:738] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.960726  2025 raft_consensus.cc:513] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.961895   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:32.963061   920 leader_election.cc:343] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.963857  1234 tablet_bootstrap.cc:492] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:32.963778  1863 raft_consensus.cc:2802] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:32.964195  1234 ts_tablet_manager.cc:1397] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.038s	user 0.006s	sys 0.005s
I20250628 07:52:32.964166  1863 raft_consensus.cc:491] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:32.964511  1863 raft_consensus.cc:3058] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.966401  1234 raft_consensus.cc:357] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.966897  1234 raft_consensus.cc:383] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:32.966785  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:32.967097  1234 raft_consensus.cc:738] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:32.967203  1122 raft_consensus.cc:3058] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:32.967682  1234 consensus_queue.cc:260] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.969699  1234 ts_tablet_manager.cc:1428] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.005s	user 0.003s	sys 0.002s
I20250628 07:52:32.969333  1229 consensus_queue.cc:260] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.972898  1234 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:32.975293  1122 raft_consensus.cc:2466] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:32.976183   758 leader_election.cc:304] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:32.977259  1928 raft_consensus.cc:2802] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:32.982098  1229 ts_tablet_manager.cc:1428] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.025s	user 0.006s	sys 0.000s
I20250628 07:52:32.982254  1234 tablet_bootstrap.cc:654] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:32.983825  1229 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:32.987839  2025 leader_election.cc:290] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:32.989537   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c4fcefaa9e954b52bbbee6e0a2f2f939" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:32.992110   757 leader_election.cc:343] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:32.995632  1863 raft_consensus.cc:513] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.994084  1230 consensus_queue.cc:260] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:32.994561  1229 tablet_bootstrap.cc:654] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.005857  1928 raft_consensus.cc:695] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:33.007958  1863 leader_election.cc:290] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.008587  1234 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:33.008980  1234 ts_tablet_manager.cc:1397] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.036s	user 0.011s	sys 0.001s
I20250628 07:52:33.009730   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:33.010423   821 raft_consensus.cc:3058] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.014110  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bed1ea07e3d44c08adf37ced9dbef85e" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:33.014667  1126 raft_consensus.cc:3058] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.012923  1234 raft_consensus.cc:357] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.016455  1234 raft_consensus.cc:383] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.016717  1234 raft_consensus.cc:738] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.017309  1234 consensus_queue.cc:260] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.019145   821 raft_consensus.cc:2466] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:33.021522  2025 consensus_queue.cc:1035] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.022970   920 leader_election.cc:304] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:33.024204  2117 raft_consensus.cc:2802] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:33.024575  2117 raft_consensus.cc:695] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:33.025333  1234 ts_tablet_manager.cc:1428] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.016s	user 0.004s	sys 0.001s
I20250628 07:52:33.025236  2117 consensus_queue.cc:237] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.026057  1229 tablet_bootstrap.cc:492] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:33.026147  1234 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:33.026592  2138 consensus_queue.cc:1035] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.027359  1229 ts_tablet_manager.cc:1397] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.045s	user 0.010s	sys 0.000s
I20250628 07:52:33.006630  1928 consensus_queue.cc:237] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.030033  1229 raft_consensus.cc:357] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.041466  1234 tablet_bootstrap.cc:654] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.042623  2025 consensus_queue.cc:1035] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:33.034649   921 consensus_peers.cc:489] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:33.045678  1229 raft_consensus.cc:383] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.046099  1229 raft_consensus.cc:738] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.047106  1229 consensus_queue.cc:260] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.050752  1230 ts_tablet_manager.cc:1428] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.134s	user 0.004s	sys 0.000s
I20250628 07:52:33.051614  1230 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:33.052816  2138 raft_consensus.cc:491] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.053265  2138 raft_consensus.cc:513] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.055884  2138 leader_election.cc:290] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.057154   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.023396  1126 raft_consensus.cc:2466] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:33.063287  1234 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:33.063673  1234 ts_tablet_manager.cc:1397] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.038s	user 0.005s	sys 0.003s
I20250628 07:52:33.068337  1229 ts_tablet_manager.cc:1428] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.041s	user 0.006s	sys 0.000s
I20250628 07:52:33.068717  1234 raft_consensus.cc:357] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.074496  1234 raft_consensus.cc:383] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.074855  1234 raft_consensus.cc:738] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.077839  1234 consensus_queue.cc:260] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.077965   955 tablet_service.cc:1468] Processing CreateTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.081426   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 670f91ee94f7460caa0790e232ac7fa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.081380  1229 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:33.083485  1234 ts_tablet_manager.cc:1428] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.017s	user 0.004s	sys 0.000s
I20250628 07:52:33.084250  1234 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:33.086975  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "132cd82d39e84e85a37f21ae208fd165" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.088979   921 leader_election.cc:304] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:33.090515  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.091471  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 670f91ee94f7460caa0790e232ac7fa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.092365  2138 raft_consensus.cc:2747] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.036551  1863 raft_consensus.cc:491] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.094677  1234 tablet_bootstrap.cc:654] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.094866   961 tablet_service.cc:1468] Processing CreateTablet for tablet b759dcf28b784ab99b7aed5d22c651dc (DEFAULT_TABLE table=default.Table_958abb2126a34f57af7942df2b6bbb9a [id=1234c3ae728a4b9fb829fed377c84833]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.095685   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b759dcf28b784ab99b7aed5d22c651dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.097244   797 tablet_service.cc:1468] Processing CreateTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.098181   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 670f91ee94f7460caa0790e232ac7fa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.100625  2025 consensus_queue.cc:1035] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.102298   799 tablet_service.cc:1468] Processing CreateTablet for tablet b759dcf28b784ab99b7aed5d22c651dc (DEFAULT_TABLE table=default.Table_958abb2126a34f57af7942df2b6bbb9a [id=1234c3ae728a4b9fb829fed377c84833]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.103390   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b759dcf28b784ab99b7aed5d22c651dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.105285  1863 raft_consensus.cc:513] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.108124  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.039996   983 raft_consensus.cc:3058] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.113288  1863 leader_election.cc:290] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.113842   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0dffb3eb778946869b914e5b05a67a69" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.118763  1234 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:33.119163  1234 ts_tablet_manager.cc:1397] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.035s	user 0.007s	sys 0.002s
I20250628 07:52:33.120457   920 leader_election.cc:304] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
W20250628 07:52:33.121191  2158 log.cc:927] Time spent T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Append to log took a long time: real 0.078s	user 0.000s	sys 0.001s
I20250628 07:52:33.121265  1234 raft_consensus.cc:357] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.121902  1234 raft_consensus.cc:383] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.122669  2167 raft_consensus.cc:2747] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.122143  1234 raft_consensus.cc:738] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.130482  1234 consensus_queue.cc:260] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.136270   961 tablet_service.cc:1468] Processing CreateTablet for tablet f026cebba0814da3acbc26669d0211d4 (DEFAULT_TABLE table=default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de [id=82f7f172b18f4b59afcc216ff724acc3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.136184  1100 tablet_service.cc:1468] Processing CreateTablet for tablet b759dcf28b784ab99b7aed5d22c651dc (DEFAULT_TABLE table=default.Table_958abb2126a34f57af7942df2b6bbb9a [id=1234c3ae728a4b9fb829fed377c84833]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.137212   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f026cebba0814da3acbc26669d0211d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.137252  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b759dcf28b784ab99b7aed5d22c651dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.138763   956 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba579ef503c427295b2ba6185aa3ccb (DEFAULT_TABLE table=default.Table_c97313f05be54e4da2bc4cbbc04c8682 [id=b98b8985d5ad44b1a1618aa721533c5c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.139147   800 tablet_service.cc:1468] Processing CreateTablet for tablet f026cebba0814da3acbc26669d0211d4 (DEFAULT_TABLE table=default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de [id=82f7f172b18f4b59afcc216ff724acc3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.139636   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba579ef503c427295b2ba6185aa3ccb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.140015   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f026cebba0814da3acbc26669d0211d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.146029  1106 tablet_service.cc:1468] Processing CreateTablet for tablet f026cebba0814da3acbc26669d0211d4 (DEFAULT_TABLE table=default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de [id=82f7f172b18f4b59afcc216ff724acc3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.147011  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f026cebba0814da3acbc26669d0211d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.148392  2117 raft_consensus.cc:491] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.148759  2117 raft_consensus.cc:513] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.152813   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.153810  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "cc0bf0edebb140209531ccf917fd1ed8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.154448  1124 raft_consensus.cc:2391] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
I20250628 07:52:33.155843   920 leader_election.cc:304] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:33.156610  2117 leader_election.cc:290] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.157217  2117 raft_consensus.cc:3058] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.162478   984 raft_consensus.cc:3058] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.164125   801 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba579ef503c427295b2ba6185aa3ccb (DEFAULT_TABLE table=default.Table_c97313f05be54e4da2bc4cbbc04c8682 [id=b98b8985d5ad44b1a1618aa721533c5c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.165043   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba579ef503c427295b2ba6185aa3ccb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.171332  1928 raft_consensus.cc:491] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.171756  1928 raft_consensus.cc:513] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.174108   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.175743  1928 leader_election.cc:290] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:33.175925   757 leader_election.cc:343] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.175958  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.176496  1124 raft_consensus.cc:2466] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:33.179986   758 leader_election.cc:304] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.181365   545 catalog_manager.cc:5582] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:33.184032  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 9ba579ef503c427295b2ba6185aa3ccb (DEFAULT_TABLE table=default.Table_c97313f05be54e4da2bc4cbbc04c8682 [id=b98b8985d5ad44b1a1618aa721533c5c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.184988  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ba579ef503c427295b2ba6185aa3ccb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.192153   963 tablet_service.cc:1468] Processing CreateTablet for tablet 51a44ec161684f74b228830ff06f1957 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.193092   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51a44ec161684f74b228830ff06f1957. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.196983   798 tablet_service.cc:1468] Processing CreateTablet for tablet 51a44ec161684f74b228830ff06f1957 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.198355   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51a44ec161684f74b228830ff06f1957. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.203238  2162 raft_consensus.cc:491] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.203830  2162 raft_consensus.cc:513] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.205269  1928 raft_consensus.cc:2802] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:33.205617  1928 raft_consensus.cc:491] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:33.206385  1928 raft_consensus.cc:3058] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.206629  2162 leader_election.cc:290] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.209288   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.209400   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:33.210381  1062 leader_election.cc:343] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:33.211158  1062 leader_election.cc:343] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.211473  1062 leader_election.cc:304] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.211997  1230 tablet_bootstrap.cc:654] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.212710  2162 raft_consensus.cc:2747] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.224074  2162 raft_consensus.cc:491] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.225920  2117 raft_consensus.cc:2747] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.226132  2162 raft_consensus.cc:513] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.227221  2138 raft_consensus.cc:491] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.227659  2138 raft_consensus.cc:513] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.235419  2138 leader_election.cc:290] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.228510  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 51a44ec161684f74b228830ff06f1957 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.236383  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.236838  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51a44ec161684f74b228830ff06f1957. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.238082  2177 consensus_queue.cc:1035] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.239445  1234 ts_tablet_manager.cc:1428] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.120s	user 0.005s	sys 0.001s
I20250628 07:52:33.241150  1234 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:33.258740   964 tablet_service.cc:1468] Processing CreateTablet for tablet b69bbcd5fede44ed9c4e1a2c35338d56 (DEFAULT_TABLE table=default.Table_5c036fde993047db95ee90d9ffd4be8d [id=640ae94271994216b7cbf2ffdef1cec9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.259672   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b69bbcd5fede44ed9c4e1a2c35338d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.259888  1107 tablet_service.cc:1468] Processing CreateTablet for tablet b69bbcd5fede44ed9c4e1a2c35338d56 (DEFAULT_TABLE table=default.Table_5c036fde993047db95ee90d9ffd4be8d [id=640ae94271994216b7cbf2ffdef1cec9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.260815  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b69bbcd5fede44ed9c4e1a2c35338d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.261528   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:33.261773   957 tablet_service.cc:1468] Processing CreateTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 (DEFAULT_TABLE table=default.Table_af13e11b58c24735a4b8dbb9a7fefa22 [id=77fa0b0008ba44c89526197767f786e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.262552   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:33.262444  1104 tablet_service.cc:1468] Processing CreateTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 (DEFAULT_TABLE table=default.Table_af13e11b58c24735a4b8dbb9a7fefa22 [id=77fa0b0008ba44c89526197767f786e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.262814   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c00063b47e49099a2c0ab9c8587c04. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.263384  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c00063b47e49099a2c0ab9c8587c04. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.263513   545 catalog_manager.cc:5582] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:33.264617   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:33.265663   962 tablet_service.cc:1468] Processing CreateTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 (DEFAULT_TABLE table=default.Table_79306d05ef4945db94f6c1fbf588aa9d [id=9c4b6cdd60804a439cbe97f8aeaa497f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.266541   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fe2e8156cc43d7b3c02092877063b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.292326  1928 raft_consensus.cc:513] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.293617   963 tablet_service.cc:1468] Processing CreateTablet for tablet 71c83c726809488ea0be8f724189e8e2 (DEFAULT_TABLE table=default.Table_123910cc7356492f93d0e1d1df124ad9 [id=5a391fa6a42f4b828a6eb4cc5e76465c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.293709  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 (DEFAULT_TABLE table=default.Table_79306d05ef4945db94f6c1fbf588aa9d [id=9c4b6cdd60804a439cbe97f8aeaa497f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.294610   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c83c726809488ea0be8f724189e8e2. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:33.294832   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 66935.6us
I20250628 07:52:33.295461  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fe2e8156cc43d7b3c02092877063b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.296363   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "58e7ec1f221f41ca8f6b019faeec3377" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.301023   798 tablet_service.cc:1468] Processing CreateTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 (DEFAULT_TABLE table=default.Table_79306d05ef4945db94f6c1fbf588aa9d [id=9c4b6cdd60804a439cbe97f8aeaa497f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.301980   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fe2e8156cc43d7b3c02092877063b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.306421   801 tablet_service.cc:1468] Processing CreateTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 (DEFAULT_TABLE table=default.Table_af13e11b58c24735a4b8dbb9a7fefa22 [id=77fa0b0008ba44c89526197767f786e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.307286   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0c00063b47e49099a2c0ab9c8587c04. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.308213  1234 tablet_bootstrap.cc:654] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.311214   797 tablet_service.cc:1468] Processing CreateTablet for tablet b69bbcd5fede44ed9c4e1a2c35338d56 (DEFAULT_TABLE table=default.Table_5c036fde993047db95ee90d9ffd4be8d [id=640ae94271994216b7cbf2ffdef1cec9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.312136   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b69bbcd5fede44ed9c4e1a2c35338d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.307132  1928 leader_election.cc:290] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.305532   920 leader_election.cc:304] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:33.322224   985 raft_consensus.cc:3058] T 132cd82d39e84e85a37f21ae208fd165 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.323246  2117 raft_consensus.cc:2747] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:33.323969  2117 consensus_queue.cc:1035] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.325552  2184 tablet.cc:1722] T febd000d51e2480e81766b292304b4f5 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:33.328079  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 71c83c726809488ea0be8f724189e8e2 (DEFAULT_TABLE table=default.Table_123910cc7356492f93d0e1d1df124ad9 [id=5a391fa6a42f4b828a6eb4cc5e76465c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.329237  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c83c726809488ea0be8f724189e8e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.346143  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:33.347774  1928 consensus_queue.cc:1035] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.348031   800 tablet_service.cc:1468] Processing CreateTablet for tablet 71c83c726809488ea0be8f724189e8e2 (DEFAULT_TABLE table=default.Table_123910cc7356492f93d0e1d1df124ad9 [id=5a391fa6a42f4b828a6eb4cc5e76465c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.346681  1122 raft_consensus.cc:3058] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.350471   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c83c726809488ea0be8f724189e8e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.356554   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ee03996105074308bda56d36ba3a1c5d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.357113   818 raft_consensus.cc:2466] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:33.358062   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "83ffa65248fc4db3aecc5f91b648b83f" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:33.359113  1062 leader_election.cc:304] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:33.359941  2161 raft_consensus.cc:2802] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:33.360323  2161 raft_consensus.cc:491] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:33.360685  2161 raft_consensus.cc:3058] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.367703  2161 raft_consensus.cc:513] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.370620  2161 leader_election.cc:290] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.372638   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ee03996105074308bda56d36ba3a1c5d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:33.373157   818 raft_consensus.cc:3058] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.373986  2162 leader_election.cc:290] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.379159   818 raft_consensus.cc:2466] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:33.384658  1062 leader_election.cc:304] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:33.357936   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ee03996105074308bda56d36ba3a1c5d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.374138   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ee03996105074308bda56d36ba3a1c5d" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:33.379016   981 raft_consensus.cc:3058] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:33.386127  1062 leader_election.cc:343] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:33.387135  1062 leader_election.cc:343] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:33.388252   757 leader_election.cc:343] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.398698  2162 raft_consensus.cc:2802] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:33.399392  2162 raft_consensus.cc:695] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:33.400389  2162 consensus_queue.cc:237] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:33.405267   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 59387.2us
I20250628 07:52:33.405864  2161 raft_consensus.cc:491] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.406520  2161 raft_consensus.cc:513] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.408767  2161 leader_election.cc:290] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.410254   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:33.423166   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]). Expected version 2 got 0
I20250628 07:52:33.433751  1234 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:33.434227  1234 ts_tablet_manager.cc:1397] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.194s	user 0.009s	sys 0.000s
I20250628 07:52:33.438163   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:33.439590  1062 leader_election.cc:343] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.445461  1928 raft_consensus.cc:491] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.445901  1928 raft_consensus.cc:513] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.447491  1928 leader_election.cc:290] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.449891  1230 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:33.450343  1230 ts_tablet_manager.cc:1397] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.399s	user 0.011s	sys 0.000s
I20250628 07:52:33.392937  1229 tablet_bootstrap.cc:654] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.453248   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:33.454020  1062 leader_election.cc:343] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:33.454672   757 leader_election.cc:343] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.454653  1062 leader_election.cc:304] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.455452  2161 raft_consensus.cc:2747] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.456583  2025 raft_consensus.cc:491] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.456497  1230 raft_consensus.cc:357] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.457314  1230 raft_consensus.cc:383] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.457613  1230 raft_consensus.cc:738] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.458268  1230 consensus_queue.cc:260] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.457070  2025 raft_consensus.cc:513] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.464622  2025 leader_election.cc:290] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.467419   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:33.467796   964 tablet_service.cc:1468] Processing CreateTablet for tablet 83766eca91da416797165dbb0262f9cb (DEFAULT_TABLE table=default.Table_d3053995fff94245ae716d9ea4118068 [id=e1a9ab2b162b43399279afd19dd0f093]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.468171   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:33.468742   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:33.469264   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:33.470497   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:33.477571   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
I20250628 07:52:33.478972   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:33.479213   798 tablet_service.cc:1468] Processing CreateTablet for tablet 83766eca91da416797165dbb0262f9cb (DEFAULT_TABLE table=default.Table_d3053995fff94245ae716d9ea4118068 [id=e1a9ab2b162b43399279afd19dd0f093]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.458362   799 tablet_service.cc:1468] Processing CreateTablet for tablet 5aece66627754857a0fc128c72269ea9 (DEFAULT_TABLE table=default.Table_be28b907172e4b36b346351fbc249f7a [id=575c9ff460e349f38b63afca848a22bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.480249   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83766eca91da416797165dbb0262f9cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.480787   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5aece66627754857a0fc128c72269ea9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.486586   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.487684   981 raft_consensus.cc:2466] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:33.492087  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 83766eca91da416797165dbb0262f9cb (DEFAULT_TABLE table=default.Table_d3053995fff94245ae716d9ea4118068 [id=e1a9ab2b162b43399279afd19dd0f093]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.493847  1230 ts_tablet_manager.cc:1428] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.043s	user 0.005s	sys 0.000s
I20250628 07:52:33.495638   804 tablet_service.cc:1468] Processing CreateTablet for tablet e4a1bb1f0adb4a82a199601863983dd7 (DEFAULT_TABLE table=default.Table_affcc9f556b943238389e6f1e205745b [id=070d78e5d6f54ab9a72d680f38247ac1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.496532   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4a1bb1f0adb4a82a199601863983dd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.462013  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 5aece66627754857a0fc128c72269ea9 (DEFAULT_TABLE table=default.Table_be28b907172e4b36b346351fbc249f7a [id=575c9ff460e349f38b63afca848a22bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.480545  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.498818  1125 raft_consensus.cc:2466] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:33.485366  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.500020  1126 raft_consensus.cc:2466] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:33.500138   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet c84abdf28d7b4c079d3e6daee9327030 consensus state has no leader
I20250628 07:52:33.500285   758 leader_election.cc:304] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:33.500571   540 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 329) took 3830 ms. Trace:
I20250628 07:52:33.500941  1928 raft_consensus.cc:2802] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:33.501259  1928 raft_consensus.cc:491] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:33.501533  1928 raft_consensus.cc:3058] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.502233   962 tablet_service.cc:1468] Processing CreateTablet for tablet e4a1bb1f0adb4a82a199601863983dd7 (DEFAULT_TABLE table=default.Table_affcc9f556b943238389e6f1e205745b [id=070d78e5d6f54ab9a72d680f38247ac1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.503235   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4a1bb1f0adb4a82a199601863983dd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.500855   540 rpcz_store.cc:276] 0628 07:52:29.670123 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.722715 (+ 52592us) service_pool.cc:225] Handling call
0628 07:52:29.834637 (+111922us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:29.835427 (+   790us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:29.836023 (+   596us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:29.836262 (+   239us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:29.943553 (+107291us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:33.500510 (+3556957us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:29.850460 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:29.850886 (+   426us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:29.850910 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:29.850920 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:29.851174 (+   254us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:29.851208 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:29.851233 (+    25us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:29.851376 (+   143us) tablet.cc:699] Row locks acquired
0628 07:52:29.851389 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:29.851500 (+   111us) write_op.cc:270] Start()
0628 07:52:29.851587 (+    87us) write_op.cc:276] Timestamp: P: 1751097149851390 usec, L: 0
0628 07:52:29.851608 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:29.894132 (+ 42524us) log.cc:844] Serialized 405 byte log entry
0628 07:52:29.895793 (+  1661us) spinlock_profiling.cc:243] Waited 42 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:29.939670 (+ 43877us) op_driver.cc:464] REPLICATION: finished
0628 07:52:29.941291 (+  1621us) write_op.cc:301] APPLY: starting
0628 07:52:29.941344 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:29.941429 (+    85us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:29.941446 (+    17us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:29.941620 (+   174us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:29.941739 (+   119us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:29.941783 (+    44us) write_op.cc:312] APPLY: finished
0628 07:52:29.942696 (+   913us) log.cc:844] Serialized 23 byte log entry
0628 07:52:29.942846 (+   150us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.942927 (+    81us) write_op.cc:454] Released schema lock
0628 07:52:29.942960 (+    33us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:29.943163 (+   203us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:29.943188 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1430,"apply.run_cpu_time_us":1146,"apply.run_wall_time_us":1869,"num_ops":1,"prepare.queue_time_us":107,"prepare.run_cpu_time_us":3958,"prepare.run_wall_time_us":45513,"raft.queue_time_us":5579,"raft.run_cpu_time_us":701,"raft.run_wall_time_us":701,"replication_time_us":87967,"spinlock_wait_cycles":96562560}]]}
I20250628 07:52:33.507956  1928 raft_consensus.cc:513] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.509621   758 leader_election.cc:304] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.510088  2147 raft_consensus.cc:491] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.511718  2147 raft_consensus.cc:513] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.512311  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83766eca91da416797165dbb0262f9cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.513442  2147 leader_election.cc:290] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.513904  2177 raft_consensus.cc:2802] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:33.514050   963 tablet_service.cc:1468] Processing CreateTablet for tablet 5aece66627754857a0fc128c72269ea9 (DEFAULT_TABLE table=default.Table_be28b907172e4b36b346351fbc249f7a [id=575c9ff460e349f38b63afca848a22bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.514235  2177 raft_consensus.cc:491] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:33.514510  2177 raft_consensus.cc:3058] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.515017   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5aece66627754857a0fc128c72269ea9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.517706  1122 raft_consensus.cc:2466] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:33.436738  1234 raft_consensus.cc:357] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.518978  1234 raft_consensus.cc:383] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.519224  1234 raft_consensus.cc:738] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.519824  1234 consensus_queue.cc:260] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.523255   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 83766eca91da416797165dbb0262f9cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.530288   758 leader_election.cc:304] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.532639  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 85b3d2d151b549d6967e0da2a57723f0 (DEFAULT_TABLE table=default.Table_db0904127a6144d7acb537f17d6891d6 [id=f8b5a176637544b0b21ec14c5bb5598f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.533576  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85b3d2d151b549d6967e0da2a57723f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.535116  1928 leader_election.cc:290] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.535595  1928 raft_consensus.cc:2802] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:33.536821  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:33.537523  1126 raft_consensus.cc:3058] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.537731  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 0e96d996085d4091b966ee51b02c140b (DEFAULT_TABLE table=default.Table_577ffb94469047e2a14ccb8d89c9a4d6 [id=3e63dedd51ba4d36bd5f7d53a2dd94e4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.538632  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e96d996085d4091b966ee51b02c140b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.541725  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5aece66627754857a0fc128c72269ea9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.543598   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8feee3be1a664cae9da417b190163a94" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:33.544130   981 raft_consensus.cc:3058] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.548313   800 tablet_service.cc:1468] Processing CreateTablet for tablet 0e96d996085d4091b966ee51b02c140b (DEFAULT_TABLE table=default.Table_577ffb94469047e2a14ccb8d89c9a4d6 [id=3e63dedd51ba4d36bd5f7d53a2dd94e4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.549319   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e96d996085d4091b966ee51b02c140b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.550168   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.551959   955 tablet_service.cc:1468] Processing CreateTablet for tablet 0e96d996085d4091b966ee51b02c140b (DEFAULT_TABLE table=default.Table_577ffb94469047e2a14ccb8d89c9a4d6 [id=3e63dedd51ba4d36bd5f7d53a2dd94e4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.552978   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e96d996085d4091b966ee51b02c140b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.556560  1107 tablet_service.cc:1468] Processing CreateTablet for tablet c84abdf28d7b4c079d3e6daee9327030 (DEFAULT_TABLE table=default.Table_a011102f62ad4163af1c9aa371b6b29f [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.557727  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84abdf28d7b4c079d3e6daee9327030. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.558763  1234 ts_tablet_manager.cc:1428] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.124s	user 0.006s	sys 0.000s
I20250628 07:52:33.559863  1234 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:33.561760   961 tablet_service.cc:1468] Processing CreateTablet for tablet c84abdf28d7b4c079d3e6daee9327030 (DEFAULT_TABLE table=default.Table_a011102f62ad4163af1c9aa371b6b29f [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.561935   541 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502] alter complete (version 1)
I20250628 07:52:33.562731   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84abdf28d7b4c079d3e6daee9327030. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.566227   795 tablet_service.cc:1468] Processing CreateTablet for tablet 711843da7df74ba2bb22d386bc477a38 (DEFAULT_TABLE table=default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.567286   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 711843da7df74ba2bb22d386bc477a38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.586540  1229 tablet_bootstrap.cc:492] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:33.586988  1229 ts_tablet_manager.cc:1397] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.518s	user 0.005s	sys 0.003s
I20250628 07:52:33.589200  1229 raft_consensus.cc:357] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.589773  1229 raft_consensus.cc:383] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.590114  1229 raft_consensus.cc:738] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.590754  1229 consensus_queue.cc:260] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.595132   545 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f] alter complete (version 2)
I20250628 07:52:33.551949   957 tablet_service.cc:1468] Processing CreateTablet for tablet 85b3d2d151b549d6967e0da2a57723f0 (DEFAULT_TABLE table=default.Table_db0904127a6144d7acb537f17d6891d6 [id=f8b5a176637544b0b21ec14c5bb5598f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.597458   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85b3d2d151b549d6967e0da2a57723f0. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:33.608647   757 leader_election.cc:343] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.541954  1230 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:33.611909  2162 raft_consensus.cc:491] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.612329  2162 raft_consensus.cc:513] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.615491   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:33.617020  1062 leader_election.cc:343] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.609864   798 tablet_service.cc:1468] Processing CreateTablet for tablet 85b3d2d151b549d6967e0da2a57723f0 (DEFAULT_TABLE table=default.Table_db0904127a6144d7acb537f17d6891d6 [id=f8b5a176637544b0b21ec14c5bb5598f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.618291   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 85b3d2d151b549d6967e0da2a57723f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.618429   797 tablet_service.cc:1468] Processing CreateTablet for tablet c84abdf28d7b4c079d3e6daee9327030 (DEFAULT_TABLE table=default.Table_a011102f62ad4163af1c9aa371b6b29f [id=9c8759743f2c49cf911b12ca926da61f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.619158  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.619748  1122 raft_consensus.cc:2466] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:33.620698   758 leader_election.cc:304] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.621654  2025 raft_consensus.cc:2802] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:33.616868  1928 raft_consensus.cc:695] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:33.621963  2025 raft_consensus.cc:491] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:33.622285  2025 raft_consensus.cc:3058] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.622929  1928 consensus_queue.cc:237] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:33.629257   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 66) took 3502 ms (client timeout 1000 ms). Trace:
W20250628 07:52:33.629622   546 rpcz_store.cc:269] 0628 07:52:30.126467 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:30.139489 (+ 13022us) service_pool.cc:225] Handling call
0628 07:52:33.629192 (+3489703us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":68604,"reported_tablets":18}
I20250628 07:52:33.631604  1100 tablet_service.cc:1468] Processing CreateTablet for tablet e4a1bb1f0adb4a82a199601863983dd7 (DEFAULT_TABLE table=default.Table_affcc9f556b943238389e6f1e205745b [id=070d78e5d6f54ab9a72d680f38247ac1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.632812  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4a1bb1f0adb4a82a199601863983dd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.633558  1126 raft_consensus.cc:2466] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:33.634423   758 leader_election.cc:304] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:33.635059  2147 raft_consensus.cc:2802] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:33.617151  2177 raft_consensus.cc:513] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.636595  2147 raft_consensus.cc:695] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:33.617894   542 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_d81430dc148d442e8dbfc9029c7cc3c1 [id=e5c2cd9d2c1b44499a609cc74538868b] alter complete (version 1)
I20250628 07:52:33.637486   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:33.637861  2147 consensus_queue.cc:237] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.620702   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c84abdf28d7b4c079d3e6daee9327030. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:33.639129   757 leader_election.cc:343] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.641216  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fef70db0996f48c69574337f8f7ecf37" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:33.641656  1125 raft_consensus.cc:3058] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:33.642170   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:33.642457   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 51) took 4033 ms (client timeout 1000 ms). Trace:
W20250628 07:52:33.642751   543 rpcz_store.cc:269] 0628 07:52:29.608890 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.635618 (+ 26728us) service_pool.cc:225] Handling call
0628 07:52:33.642396 (+4006778us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":132344,"reported_tablets":16}
I20250628 07:52:33.644388   958 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:33.645926   958 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:33.647068   958 tablet_service.cc:1515] Processing DeleteTablet for tablet e28ab08f878d4d4b86a5646f687b08fc with delete_type TABLET_DATA_DELETED (replaced by tablet 711843da7df74ba2bb22d386bc477a38) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:33.647549  1125 raft_consensus.cc:2466] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:33.648589   758 leader_election.cc:304] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.649289  2180 raft_consensus.cc:2802] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:33.649699  2180 raft_consensus.cc:695] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:33.650422  2180 consensus_queue.cc:237] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.655915   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:33.656947  1062 leader_election.cc:343] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.657320  1062 leader_election.cc:304] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.657941  2161 raft_consensus.cc:2747] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:33.607757   981 raft_consensus.cc:2466] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:33.658663  1107 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:33.659080  1109 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:33.659587  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:33.661300  1103 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:33.669090  1229 ts_tablet_manager.cc:1428] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.082s	user 0.006s	sys 0.001s
I20250628 07:52:33.670073  1229 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:33.672843   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5ec8c1d9386a4922b55df5e06039858d because the tablet was not found. No further retry: Not found: Tablet not found: 5ec8c1d9386a4922b55df5e06039858d
W20250628 07:52:33.674919   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b2d49e5a06384758bce5e7328b6a10f8 because the tablet was not found. No further retry: Not found: Tablet not found: b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:33.675194   961 tablet_service.cc:1468] Processing CreateTablet for tablet 711843da7df74ba2bb22d386bc477a38 (DEFAULT_TABLE table=default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:33.675459   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because the tablet was not found. No further retry: Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:33.676091   958 tablet_service.cc:1468] Processing CreateTablet for tablet 58d853d69a5145cdabb33437463d4aa2 (DEFAULT_TABLE table=default.Table_50f3c0fe4d0445c7b1cc05590d41a738 [id=4454ad1ffc2744a1b55d3772f0722258]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.676929  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 711843da7df74ba2bb22d386bc477a38 (DEFAULT_TABLE table=default.Table_8146958ca541498babe752911c4346bd [id=8a7e8a043ede48228adf10a0d7adc55a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.678104  2175 raft_consensus.cc:491] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.678617  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 711843da7df74ba2bb22d386bc477a38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.678663  2175 raft_consensus.cc:513] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.680936  2175 leader_election.cc:290] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.683197  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:33.683917  1125 raft_consensus.cc:2466] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:33.684698  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 58d853d69a5145cdabb33437463d4aa2 (DEFAULT_TABLE table=default.Table_50f3c0fe4d0445c7b1cc05590d41a738 [id=4454ad1ffc2744a1b55d3772f0722258]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.685602  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58d853d69a5145cdabb33437463d4aa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.685963   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.688586  2025 raft_consensus.cc:513] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.689534   539 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887] alter complete (version 1)
I20250628 07:52:33.690274  2025 leader_election.cc:290] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.678388  2162 leader_election.cc:290] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.693807   795 tablet_service.cc:1468] Processing CreateTablet for tablet 58d853d69a5145cdabb33437463d4aa2 (DEFAULT_TABLE table=default.Table_50f3c0fe4d0445c7b1cc05590d41a738 [id=4454ad1ffc2744a1b55d3772f0722258]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.694789   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58d853d69a5145cdabb33437463d4aa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.676126   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 711843da7df74ba2bb22d386bc477a38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.673436  2208 tablet_replica.cc:331] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:33.696903  2208 raft_consensus.cc:2241] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:33.636588  2177 leader_election.cc:290] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.698784  2208 raft_consensus.cc:2270] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:33.699110   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 58d853d69a5145cdabb33437463d4aa2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.706529  2208 ts_tablet_manager.cc:1905] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:33.711339  1230 tablet_bootstrap.cc:654] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.725761   965 tablet_service.cc:1468] Processing CreateTablet for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f (DEFAULT_TABLE table=default.Table_d34b80de26354403b4ea3b3994522090 [id=d535cd0c729146ff97bb42c7db33bbcb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.726667  1234 tablet_bootstrap.cc:654] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.726747   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.729409  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 924b921830b7403596ebcc6cf80dc24d (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.729684  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f (DEFAULT_TABLE table=default.Table_d34b80de26354403b4ea3b3994522090 [id=d535cd0c729146ff97bb42c7db33bbcb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.730360  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924b921830b7403596ebcc6cf80dc24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.730860  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:33.732985   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 58) took 3922 ms (client timeout 1000 ms). Trace:
W20250628 07:52:33.733430   538 rpcz_store.cc:269] 0628 07:52:29.810645 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:29.813763 (+  3118us) service_pool.cc:225] Handling call
0628 07:52:29.896658 (+ 82895us) spinlock_profiling.cc:243] Waited 79.7 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:33.732921 (+3836263us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:31.669465 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:31.669835 (+   370us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:31.669862 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:31.669874 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:31.670163 (+   289us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:31.670198 (+    35us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:31.670218 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:31.696526 (+ 26308us) tablet.cc:699] Row locks acquired
0628 07:52:31.696541 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:31.696673 (+   132us) write_op.cc:270] Start()
0628 07:52:31.696760 (+    87us) write_op.cc:276] Timestamp: P: 1751097151696614 usec, L: 0
0628 07:52:31.696775 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:31.697919 (+  1144us) log.cc:844] Serialized 913 byte log entry
0628 07:52:31.721598 (+ 23679us) op_driver.cc:464] REPLICATION: finished
0628 07:52:31.721915 (+   317us) write_op.cc:301] APPLY: starting
0628 07:52:31.721964 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:31.722062 (+    98us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:31.722139 (+    77us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:31.722395 (+   256us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:31.722523 (+   128us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:31.722563 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:31.722725 (+   162us) log.cc:844] Serialized 29 byte log entry
0628 07:52:31.722863 (+   138us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.722930 (+    67us) write_op.cc:454] Released schema lock
0628 07:52:31.722961 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:31.723173 (+   212us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.723195 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":69251,"reported_tablets":18,"spinlock_wait_cycles":183369600,"child_traces":[["op",{"apply.queue_time_us":181,"apply.run_cpu_time_us":1224,"apply.run_wall_time_us":1224,"num_ops":2,"prepare.queue_time_us":174519,"prepare.run_cpu_time_us":2085,"prepare.run_wall_time_us":30103,"raft.queue_time_us":76,"raft.run_cpu_time_us":582,"raft.run_wall_time_us":15027,"replication_time_us":24728}]]}
W20250628 07:52:33.736654   531 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 104034us
I20250628 07:52:33.749325  2206 tablet_replica.cc:331] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
W20250628 07:52:33.749944   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:33.750180  2206 raft_consensus.cc:2241] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:33.750651  2206 raft_consensus.cc:2270] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:33.753626   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because the tablet was not found. No further retry: Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:33.766410   956 tablet_service.cc:1468] Processing CreateTablet for tablet f0d48652a4da42f3b26b42f65c8db51b (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.767578   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0d48652a4da42f3b26b42f65c8db51b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.773643  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 4c2e91519c6b4b509d96f508546c8fc4 (DEFAULT_TABLE table=default.Table_7437789ae57b4f029d85e0722aa8fa84 [id=442671f219f54e26b6f9d8958301fbd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.774708  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4c2e91519c6b4b509d96f508546c8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.779502  1230 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:33.780063  1230 ts_tablet_manager.cc:1397] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.238s	user 0.008s	sys 0.000s
I20250628 07:52:33.780402  1101 tablet_service.cc:1468] Processing CreateTablet for tablet f0d48652a4da42f3b26b42f65c8db51b (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.782199  1230 raft_consensus.cc:357] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.782692  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0d48652a4da42f3b26b42f65c8db51b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.782769  1230 raft_consensus.cc:383] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.787325  1230 raft_consensus.cc:738] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.788026  1230 consensus_queue.cc:260] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.797999  1229 tablet_bootstrap.cc:654] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.800401  1230 ts_tablet_manager.cc:1428] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.020s	user 0.007s	sys 0.000s
I20250628 07:52:33.805145   963 tablet_service.cc:1468] Processing CreateTablet for tablet 4c2e91519c6b4b509d96f508546c8fc4 (DEFAULT_TABLE table=default.Table_7437789ae57b4f029d85e0722aa8fa84 [id=442671f219f54e26b6f9d8958301fbd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.806082   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4c2e91519c6b4b509d96f508546c8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.810673  1234 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:33.811928  1234 ts_tablet_manager.cc:1397] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.251s	user 0.008s	sys 0.000s
I20250628 07:52:33.814009  1234 raft_consensus.cc:357] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.814725  1234 raft_consensus.cc:383] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.814961  1234 raft_consensus.cc:738] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.815466  1234 consensus_queue.cc:260] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.819128   964 tablet_service.cc:1468] Processing CreateTablet for tablet 924b921830b7403596ebcc6cf80dc24d (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:33.811040  2208 ts_tablet_manager.cc:1918] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:33.830438  2208 log.cc:1199] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:33.830735   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:33.831235  2208 ts_tablet_manager.cc:1939] T 0dffb3eb778946869b914e5b05a67a69 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:33.831956   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924b921830b7403596ebcc6cf80dc24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:33.832677   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:33.798642  2206 ts_tablet_manager.cc:1905] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:33.833508  1229 tablet_bootstrap.cc:492] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:33.833909  1229 ts_tablet_manager.cc:1397] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.164s	user 0.006s	sys 0.003s
I20250628 07:52:33.837437  1230 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:33.839648   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 0dffb3eb778946869b914e5b05a67a69 (table default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]) successfully deleted
I20250628 07:52:33.845738   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]). Expected version 2 got 0
W20250628 07:52:33.879843   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 2595308cbc0d44f0abfd0a126f16d4df (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 2595308cbc0d44f0abfd0a126f16d4df consensus state has no leader
I20250628 07:52:33.882771  2162 raft_consensus.cc:491] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.837031  1229 raft_consensus.cc:357] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.883234  2162 raft_consensus.cc:513] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.883735  1229 raft_consensus.cc:383] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.884085  1229 raft_consensus.cc:738] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.884792  1229 consensus_queue.cc:260] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.886016  2162 leader_election.cc:290] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.885725   543 catalog_manager.cc:5582] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:33.886713   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:33.886986   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:33.887833  1229 ts_tablet_manager.cc:1428] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.054s	user 0.006s	sys 0.000s
W20250628 07:52:33.887902  1062 leader_election.cc:343] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.887866   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:33.888653   543 catalog_manager.cc:5582] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:33.890053   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:33.890654   543 catalog_manager.cc:5582] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:33.895779   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:33.897048   543 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:33.912145  1230 tablet_bootstrap.cc:654] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.918555  1229 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:33.923929  1229 tablet_bootstrap.cc:654] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:33.925567  1125 raft_consensus.cc:1273] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20250628 07:52:33.925647  2162 raft_consensus.cc:491] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.926054  2162 raft_consensus.cc:513] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.927057  1863 consensus_queue.cc:1035] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:33.927793  2162 leader_election.cc:290] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:33.929450   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:33.930871  1062 leader_election.cc:343] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.940351  1229 tablet_bootstrap.cc:492] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:33.940773  1229 ts_tablet_manager.cc:1397] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.022s	user 0.010s	sys 0.000s
I20250628 07:52:33.942997  1229 raft_consensus.cc:357] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.943675  1229 raft_consensus.cc:383] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:33.943915  1229 raft_consensus.cc:738] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:33.944455  1229 consensus_queue.cc:260] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.951961   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]). Expected version 2 got 0
I20250628 07:52:33.961792  1384 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.190:37209
I20250628 07:52:33.964728  1229 ts_tablet_manager.cc:1428] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.024s	user 0.006s	sys 0.000s
W20250628 07:52:33.970086   757 leader_election.cc:343] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:33.971276   758 leader_election.cc:304] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:33.972730   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:33.978961  1229 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:33.985138  1230 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:33.988932   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:33.961148  2228 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.190:37209 every 8 connection(s)
I20250628 07:52:33.996420  1863 raft_consensus.cc:491] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:33.996819  1863 raft_consensus.cc:513] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:33.998286  1863 leader_election.cc:290] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:33.991791   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:34.000327   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:34.000912   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:34.003683  2230 raft_consensus.cc:491] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:34.003823   921 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:34.004352  2230 raft_consensus.cc:513] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.007761   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 1 got 0
I20250628 07:52:34.008777   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:34.009533  1229 tablet_bootstrap.cc:654] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:34.010017   793 tablet_service.cc:1468] Processing CreateTablet for tablet f0d48652a4da42f3b26b42f65c8db51b (DEFAULT_TABLE table=default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.009692   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:34.013407   793 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f0d48652a4da42f3b26b42f65c8db51b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.016237  1230 ts_tablet_manager.cc:1397] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.179s	user 0.011s	sys 0.000s
I20250628 07:52:33.990101  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:34.017023  1125 raft_consensus.cc:3058] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.021447  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:34.021564  2230 leader_election.cc:290] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.021987  1126 raft_consensus.cc:2466] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:34.023002   758 leader_election.cc:304] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:34.036420   794 tablet_service.cc:1468] Processing CreateTablet for tablet 4c2e91519c6b4b509d96f508546c8fc4 (DEFAULT_TABLE table=default.Table_7437789ae57b4f029d85e0722aa8fa84 [id=442671f219f54e26b6f9d8958301fbd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.037732   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4c2e91519c6b4b509d96f508546c8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.045655   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.062431   797 tablet_service.cc:1468] Processing CreateTablet for tablet 924b921830b7403596ebcc6cf80dc24d (DEFAULT_TABLE table=default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.063764   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 924b921830b7403596ebcc6cf80dc24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.066926  1384 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
I20250628 07:52:34.075734   802 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.076171   800 tablet_service.cc:1468] Processing CreateTablet for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f (DEFAULT_TABLE table=default.Table_d34b80de26354403b4ea3b3994522090 [id=d535cd0c729146ff97bb42c7db33bbcb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.077363   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.078864   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 9a248bf8a62b4b4eb206c0daff5e0568 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.079541   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:34.081449   815 raft_consensus.cc:1273] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Refusing update from remote peer 82f43cdfab544b17a6a71033983e65d4: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 2 index: 1. (term mismatch)
I20250628 07:52:34.081862   815 pending_rounds.cc:85] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Aborting all ops after (but not including) 0
I20250628 07:52:34.081993   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:34.082190   815 pending_rounds.cc:107] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: Aborting uncommitted NO_OP operation due to leader change: 1.1
I20250628 07:52:34.082585   815 raft_consensus.cc:2887] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted by new leader
I20250628 07:52:34.081449   805 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.085404  1863 consensus_queue.cc:1035] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.087738  1229 tablet_bootstrap.cc:492] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:34.088198  1229 ts_tablet_manager.cc:1397] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.109s	user 0.008s	sys 0.015s
I20250628 07:52:34.090662  1229 raft_consensus.cc:357] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.091321  1229 raft_consensus.cc:383] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.091681  1229 raft_consensus.cc:738] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.092391  1229 consensus_queue.cc:260] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.097373  1125 raft_consensus.cc:2466] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:34.051367  1230 raft_consensus.cc:357] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.100844  1230 raft_consensus.cc:383] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.101131  1230 raft_consensus.cc:738] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.103528  1229 ts_tablet_manager.cc:1428] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
I20250628 07:52:34.106223  1229 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:34.098925  2206 ts_tablet_manager.cc:1918] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:34.110035  2206 log.cc:1199] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:34.110885  2206 ts_tablet_manager.cc:1939] T ef060774191146168ee8d1e13c9cd71b P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:34.113516   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet ef060774191146168ee8d1e13c9cd71b (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
I20250628 07:52:34.113445  1230 consensus_queue.cc:260] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:34.114473   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e28ab08f878d4d4b86a5646f687b08fc because the tablet was not found. No further retry: Not found: Tablet not found: e28ab08f878d4d4b86a5646f687b08fc
W20250628 07:52:34.116402  1062 leader_election.cc:343] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:34.116531   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 145202us
I20250628 07:52:34.116814  1062 leader_election.cc:304] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.085754   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b2d49e5a06384758bce5e7328b6a10f8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:34.121555  2161 raft_consensus.cc:491] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.121970  2161 raft_consensus.cc:513] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.122648  1230 ts_tablet_manager.cc:1428] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.106s	user 0.006s	sys 0.000s
I20250628 07:52:34.123482  1230 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:34.123744  2161 leader_election.cc:290] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
W20250628 07:52:34.126065  1062 leader_election.cc:343] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.126400  1062 leader_election.cc:304] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.126508   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:34.127331  2162 raft_consensus.cc:2747] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:34.127832  2162 raft_consensus.cc:2747] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:34.131803  1229 tablet_bootstrap.cc:654] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:34.133352  1062 leader_election.cc:343] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.143064  1230 tablet_bootstrap.cc:654] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:34.143157  1863 consensus_queue.cc:1035] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.157227  1229 tablet_bootstrap.cc:492] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:34.157707  1229 ts_tablet_manager.cc:1397] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.052s	user 0.010s	sys 0.000s
I20250628 07:52:34.160279  1229 raft_consensus.cc:357] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.161048  1229 raft_consensus.cc:383] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.161377  1229 raft_consensus.cc:738] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.162122  1229 consensus_queue.cc:260] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.172334   981 raft_consensus.cc:3058] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.172487   984 raft_consensus.cc:3058] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.173381   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:34.176828   538 catalog_manager.cc:5582] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 1 to 2, leader changed from 6005c44dfe4848f18a304621cd49e934 (127.0.107.129) to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 2 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:34.177899  2025 consensus_queue.cc:1035] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:34.179255  2025 raft_consensus.cc:2802] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:34.179549  2025 raft_consensus.cc:491] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:34.179774   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:34.179890  2025 raft_consensus.cc:3058] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.180789   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:34.184859  1230 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:34.185322  1230 ts_tablet_manager.cc:1397] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.062s	user 0.009s	sys 0.002s
I20250628 07:52:34.187290  1234 ts_tablet_manager.cc:1428] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.375s	user 0.005s	sys 0.000s
I20250628 07:52:34.187667   798 tablet_service.cc:1468] Processing CreateTablet for tablet 55045da381224ff28ae934ac3e28dd05 (DEFAULT_TABLE table=default.Table_7ac89842d7e54a2ea03d7f64f4a92175 [id=28124975e8e246c68bd570b13fa5af2a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.191105  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 55045da381224ff28ae934ac3e28dd05 (DEFAULT_TABLE table=default.Table_7ac89842d7e54a2ea03d7f64f4a92175 [id=28124975e8e246c68bd570b13fa5af2a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.192092  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 55045da381224ff28ae934ac3e28dd05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.194203   538 catalog_manager.cc:5582] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:34.199525  1176 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:34.201933   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:34.202685   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:34.208112  1230 raft_consensus.cc:357] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.208775  1230 raft_consensus.cc:383] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.209220   541 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_df287bcea5bd414ebe5aee1689733f8c [id=398556bc3bdc481cb6577321cbc31055] alter complete (version 1)
I20250628 07:52:34.209796   963 tablet_service.cc:1468] Processing CreateTablet for tablet 55045da381224ff28ae934ac3e28dd05 (DEFAULT_TABLE table=default.Table_7ac89842d7e54a2ea03d7f64f4a92175 [id=28124975e8e246c68bd570b13fa5af2a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:34.210772   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 55045da381224ff28ae934ac3e28dd05. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:34.209049  1230 raft_consensus.cc:738] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.215940  1230 consensus_queue.cc:260] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.219223  1230 ts_tablet_manager.cc:1428] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.000s
I20250628 07:52:34.227046  2175 raft_consensus.cc:491] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.227674  2175 raft_consensus.cc:513] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.196213   758 leader_election.cc:304] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
W20250628 07:52:34.234866   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 59) took 3391 ms (client timeout 1000 ms). Trace:
I20250628 07:52:34.243690  1230 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:34.166356  1229 ts_tablet_manager.cc:1428] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.000s
I20250628 07:52:34.173920   820 raft_consensus.cc:2466] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:34.247601  1928 raft_consensus.cc:2802] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:34.247977  1928 raft_consensus.cc:491] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:34.248268  1928 raft_consensus.cc:3058] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.253172  2238 sys_catalog.cc:263] Verifying existing consensus state
W20250628 07:52:34.255571   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 157128us
I20250628 07:52:34.257540  2231 raft_consensus.cc:2802] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:34.270112   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.267277  2237 tablet_replica.cc:331] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:34.274523   796 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.274720   795 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.274741  2180 consensus_queue.cc:1035] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.004s
I20250628 07:52:34.072978   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 1384
W20250628 07:52:34.277601   757 leader_election.cc:343] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:34.278388   757 leader_election.cc:343] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.279088   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.280813   795 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.278769  1230 tablet_bootstrap.cc:654] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:34.260432  2231 raft_consensus.cc:695] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:34.295297  2177 consensus_queue.cc:1035] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:34.306346   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:34.306814   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 55045da381224ff28ae934ac3e28dd05. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:34.310303  1062 leader_election.cc:343] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.310921  1062 leader_election.cc:304] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.312361  2161 raft_consensus.cc:2747] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:34.298857  2175 leader_election.cc:290] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.316218  1234 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:34.315791  2231 consensus_queue.cc:237] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:34.321792   758 consensus_peers.cc:489] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0dffb3eb778946869b914e5b05a67a69. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:34.322945  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:34.323673  1124 raft_consensus.cc:2466] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:34.326706   542 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 52) took 3707 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.241220   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 60) took 2127 ms (client timeout 1000 ms). Trace:
I20250628 07:52:34.331585  1928 raft_consensus.cc:513] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.334683  2238 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Bootstrap starting.
W20250628 07:52:34.340369   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 53) took 2710 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.340704   547 rpcz_store.cc:269] 0628 07:52:31.629610 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:31.758853 (+129243us) service_pool.cc:225] Handling call
0628 07:52:31.817763 (+ 58910us) spinlock_profiling.cc:243] Waited 47.5 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.340304 (+2522541us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":49738,"reported_tablets":19,"spinlock_wait_cycles":109264768}
I20250628 07:52:34.263427  1033 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:34.325168   758 leader_election.cc:304] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:34.273170   964 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.275171   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.281549   920 leader_election.cc:304] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
W20250628 07:52:34.347182   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 65706.6us
I20250628 07:52:34.347754   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af] alter complete (version 2)
I20250628 07:52:34.348860  2138 raft_consensus.cc:2802] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:34.349212  2138 raft_consensus.cc:491] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:34.349510  2138 raft_consensus.cc:3058] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.288638   958 tablet_service.cc:1515] Processing DeleteTablet for tablet 0f29c8ffb7df410295278eee9c578b03 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:18 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.311585   961 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.311842   957 tablet_service.cc:1515] Processing DeleteTablet for tablet e28ab08f878d4d4b86a5646f687b08fc with delete_type TABLET_DATA_DELETED (replaced by tablet 711843da7df74ba2bb22d386bc477a38) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.319172   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.320010   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.320103   962 tablet_service.cc:1515] Processing DeleteTablet for tablet cb62cf19aaf04444845358cc8ad611fb with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:20 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.354557  1863 consensus_queue.cc:1035] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.355718  2206 tablet_replica.cc:331] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:34.356074  1229 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:34.356629  2206 raft_consensus.cc:2241] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:34.357594  2206 raft_consensus.cc:2270] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.360225  2206 ts_tablet_manager.cc:1905] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:34.366402  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 532ba24d0d644313b9ef4e3ec10b0fed with delete_type TABLET_DATA_DELETED (replaced by tablet c84abdf28d7b4c079d3e6daee9327030) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:34.372557   540 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 54) took 1715 ms (client timeout 1000 ms). Trace:
I20250628 07:52:34.379909  2025 raft_consensus.cc:513] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.382858  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.376878  2138 raft_consensus.cc:513] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:34.389803   921 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:34.394703   795 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:34.396471   540 rpcz_store.cc:269] 0628 07:52:32.657381 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:33.571389 (+914008us) service_pool.cc:225] Handling call
0628 07:52:34.372478 (+801089us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":164232,"reported_tablets":24}
I20250628 07:52:34.398383  1234 tablet_bootstrap.cc:654] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:34.398154   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c8bc72fa0f974cfebafe4cc4b90bb008"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:34.399646   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:34.403235  1928 leader_election.cc:290] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.405390  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:34.405493  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.405946  1124 raft_consensus.cc:3058] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:34.262866   539 rpcz_store.cc:269] 0628 07:52:30.843274 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:30.922979 (+ 79705us) service_pool.cc:225] Handling call
0628 07:52:34.234803 (+3311824us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:32.222905 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:32.223271 (+   366us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:32.223292 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:32.223304 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:32.223535 (+   231us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:32.223571 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:32.223593 (+    22us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:32.223720 (+   127us) tablet.cc:699] Row locks acquired
0628 07:52:32.223732 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:32.223824 (+    92us) write_op.cc:270] Start()
0628 07:52:32.223913 (+    89us) write_op.cc:276] Timestamp: P: 1751097152223773 usec, L: 0
0628 07:52:32.223928 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:32.224392 (+   464us) log.cc:844] Serialized 533 byte log entry
0628 07:52:32.235611 (+ 11219us) op_driver.cc:464] REPLICATION: finished
0628 07:52:32.235974 (+   363us) write_op.cc:301] APPLY: starting
0628 07:52:32.236028 (+    54us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:32.236109 (+    81us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:32.236193 (+    84us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:32.236374 (+   181us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:32.236489 (+   115us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:32.236527 (+    38us) write_op.cc:312] APPLY: finished
0628 07:52:32.236682 (+   155us) log.cc:844] Serialized 23 byte log entry
0628 07:52:32.236814 (+   132us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:32.236880 (+    66us) write_op.cc:454] Released schema lock
0628 07:52:32.236909 (+    29us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:32.273656 (+ 36747us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:32.273682 (+    26us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:33.664416 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.664799 (+   383us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.664827 (+    28us) write_op.cc:435] Acquired schema lock
0628 07:52:33.664839 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:33.665110 (+   271us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.665150 (+    40us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.665172 (+    22us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:33.665301 (+   129us) tablet.cc:699] Row locks acquired
0628 07:52:33.665314 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:33.665411 (+    97us) write_op.cc:270] Start()
0628 07:52:33.665486 (+    75us) write_op.cc:276] Timestamp: P: 1751097153665364 usec, L: 0
0628 07:52:33.665501 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.665965 (+   464us) log.cc:844] Serialized 356 byte log entry
0628 07:52:33.680368 (+ 14403us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.680644 (+   276us) write_op.cc:301] APPLY: starting
0628 07:52:33.680695 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.680796 (+   101us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.680876 (+    80us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.681057 (+   181us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.681198 (+   141us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:33.681240 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:33.681411 (+   171us) log.cc:844] Serialized 23 byte log entry
0628 07:52:33.681534 (+   123us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.681594 (+    60us) write_op.cc:454] Released schema lock
0628 07:52:33.681625 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:33.681836 (+   211us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.681857 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":32756,"reported_tablets":22,"child_traces":[["op",{"apply.queue_time_us":213,"apply.run_cpu_time_us":1112,"apply.run_wall_time_us":1112,"num_ops":1,"prepare.queue_time_us":92213,"prepare.run_cpu_time_us":1786,"prepare.run_wall_time_us":50740,"raft.queue_time_us":85,"raft.run_cpu_time_us":655,"raft.run_wall_time_us":656,"replication_time_us":11591}],["op",{"apply.queue_time_us":144,"apply.run_cpu_time_us":1148,"apply.run_wall_time_us":1148,"num_ops":1,"prepare.queue_time_us":1992,"prepare.run_cpu_time_us":1819,"prepare.run_wall_time_us":4300,"raft.queue_time_us":78,"raft.run_cpu_time_us":1143,"raft.run_wall_time_us":1142,"replication_time_us":14776}]]}
I20250628 07:52:34.407974   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a5aaafa5f2c84037b58a9040e19581dd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:34.417964  2025 leader_election.cc:290] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.419346  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:34.419831  1126 raft_consensus.cc:3058] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.418519  2237 raft_consensus.cc:2241] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:34.422273  2237 raft_consensus.cc:2270] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:34.422956   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 61) took 935 ms (client timeout 1000 ms). Trace:
I20250628 07:52:34.424582  2237 ts_tablet_manager.cc:1905] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:34.434222   794 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.434499  2161 consensus_queue.cc:1035] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.434895  1126 raft_consensus.cc:2466] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:34.435585  1124 raft_consensus.cc:2466] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:34.436301  2248 raft_consensus.cc:2802] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:34.436692  2248 raft_consensus.cc:491] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:34.441110  2248 raft_consensus.cc:3058] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.441926   758 leader_election.cc:304] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:34.443404   794 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.443686  2025 raft_consensus.cc:2802] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:34.444254  2025 raft_consensus.cc:695] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:34.444880   794 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.445035  2025 consensus_queue.cc:237] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.446318   794 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.448962   758 leader_election.cc:304] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:34.453172  1234 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:34.471213  1234 ts_tablet_manager.cc:1397] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.155s	user 0.010s	sys 0.003s
I20250628 07:52:34.473872   874 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:34.471201  1928 raft_consensus.cc:2802] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:34.476336  1928 raft_consensus.cc:695] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:34.477361  1928 consensus_queue.cc:237] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.474093  1234 raft_consensus.cc:357] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.485038  1234 raft_consensus.cc:383] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.485419  1234 raft_consensus.cc:738] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.485992  1234 consensus_queue.cc:260] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.487520  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:34.490067  1062 consensus_peers.cc:489] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:34.487329  2237 ts_tablet_manager.cc:1918] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:34.490774  2237 log.cc:1199] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/5ec8c1d9386a4922b55df5e06039858d
I20250628 07:52:34.491721  2237 ts_tablet_manager.cc:1939] T 5ec8c1d9386a4922b55df5e06039858d P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:34.492727  1234 ts_tablet_manager.cc:1428] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.021s	user 0.006s	sys 0.000s
I20250628 07:52:34.492765  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.494254   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 5ec8c1d9386a4922b55df5e06039858d (table default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]) successfully deleted
I20250628 07:52:34.494482  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:34.495824   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet cb62cf19aaf04444845358cc8ad611fb because the tablet was not found. No further retry: Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
W20250628 07:52:34.496358   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 67) took 3330 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.497578   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 9a248bf8a62b4b4eb206c0daff5e0568 because the tablet was not found. No further retry: Not found: Tablet not found: 9a248bf8a62b4b4eb206c0daff5e0568
I20250628 07:52:34.493697  1234 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:34.505242   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 56361.9us
W20250628 07:52:34.505394   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet ef060774191146168ee8d1e13c9cd71b because the tablet was not found. No further retry: Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:34.508738  1234 tablet_bootstrap.cc:654] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:34.516222   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 532ba24d0d644313b9ef4e3ec10b0fed because the tablet was not found. No further retry: Not found: Tablet not found: 532ba24d0d644313b9ef4e3ec10b0fed
W20250628 07:52:34.517302   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 532ba24d0d644313b9ef4e3ec10b0fed because the tablet was not found. No further retry: Not found: Tablet not found: 532ba24d0d644313b9ef4e3ec10b0fed
W20250628 07:52:34.517442   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 238096us
I20250628 07:52:34.518201  2237 tablet_replica.cc:331] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:34.518389  2177 consensus_queue.cc:1035] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.519625  2237 raft_consensus.cc:2241] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.522969   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:34.525017   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because the tablet was not found. No further retry: Not found: Tablet not found: 0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:34.520100  2237 raft_consensus.cc:2270] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:34.330946   545 rpcz_store.cc:269] 0628 07:52:32.113334 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:32.182179 (+ 68845us) service_pool.cc:225] Handling call
0628 07:52:32.211360 (+ 29181us) spinlock_profiling.cc:243] Waited 17.9 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:33.321172 (+1109812us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.321195 (+    23us) write_op.cc:454] Released schema lock
0628 07:52:34.241155 (+919960us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:33.293319 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.293640 (+   321us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.293665 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:33.293676 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:33.293944 (+   268us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.293984 (+    40us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.294005 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:33.294213 (+   208us) tablet.cc:699] Row locks acquired
0628 07:52:33.294225 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:33.294322 (+    97us) write_op.cc:270] Start()
0628 07:52:33.294393 (+    71us) write_op.cc:276] Timestamp: P: 1751097153294263 usec, L: 0
0628 07:52:33.294411 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.294857 (+   446us) log.cc:844] Serialized 913 byte log entry
0628 07:52:33.305325 (+ 10468us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.306355 (+  1030us) write_op.cc:301] APPLY: starting
0628 07:52:33.306420 (+    65us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.306525 (+   105us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.306596 (+    71us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.306898 (+   302us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.307014 (+   116us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:33.307052 (+    38us) write_op.cc:312] APPLY: finished
0628 07:52:33.307211 (+   159us) log.cc:844] Serialized 29 byte log entry
0628 07:52:33.307337 (+   126us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.307406 (+    69us) write_op.cc:454] Released schema lock
0628 07:52:33.307441 (+    35us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:33.564650 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.565041 (+   391us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.565070 (+    29us) write_op.cc:435] Acquired schema lock
0628 07:52:33.565103 (+    33us) tablet.cc:660] Decoding operations
0628 07:52:33.565354 (+   251us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.565391 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.565412 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:33.565535 (+   123us) tablet.cc:699] Row locks acquired
0628 07:52:33.565546 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:33.565652 (+   106us) write_op.cc:270] Start()
0628 07:52:33.565724 (+    72us) write_op.cc:276] Timestamp: P: 1751097153565604 usec, L: 0
0628 07:52:33.565742 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.566170 (+   428us) log.cc:844] Serialized 356 byte log entry
0628 07:52:33.578785 (+ 12615us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.579082 (+   297us) write_op.cc:301] APPLY: starting
0628 07:52:33.579133 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.579211 (+    78us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.579288 (+    77us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.579449 (+   161us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.579567 (+   118us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:33.579606 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:33.579761 (+   155us) log.cc:844] Serialized 23 byte log entry
0628 07:52:33.579901 (+   140us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.579957 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:33.579984 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:33.580174 (+   190us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.580202 (+    28us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":40924,"reported_tablets":25,"spinlock_wait_cycles":41209728,"child_traces":[["op",{"apply.queue_time_us":892,"apply.run_cpu_time_us":1265,"apply.run_wall_time_us":1264,"num_ops":2,"prepare.queue_time_us":9108,"prepare.run_cpu_time_us":1770,"prepare.run_wall_time_us":1771,"raft.queue_time_us":1134,"raft.run_cpu_time_us":619,"raft.run_wall_time_us":619,"replication_time_us":10825}],["op",{"apply.queue_time_us":175,"apply.run_cpu_time_us":1067,"apply.run_wall_time_us":1067,"num_ops":1,"prepare.queue_time_us":89,"prepare.run_cpu_time_us":1747,"prepare.run_wall_time_us":1747,"replication_time_us":12956}]]}
I20250628 07:52:34.530053   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_52441cbec305498ebed774e69004a383" } modify_external_catalogs: true
I20250628 07:52:34.530575  2138 leader_election.cc:290] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.530623   984 raft_consensus.cc:3058] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.530701   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "bfc303c4588d44428984d8203fc05185" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:34.531203   986 raft_consensus.cc:3058] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:34.327035   542 rpcz_store.cc:269] 0628 07:52:30.619436 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:30.720974 (+101538us) service_pool.cc:225] Handling call
0628 07:52:30.789573 (+ 68599us) spinlock_profiling.cc:243] Waited 65.6 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.310330 (+3520757us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:31.374148 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:31.374543 (+   395us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:31.374566 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:31.374577 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:31.413956 (+ 39379us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:31.413992 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:31.414013 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:31.414272 (+   259us) tablet.cc:699] Row locks acquired
0628 07:52:31.414283 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:31.414382 (+    99us) write_op.cc:270] Start()
0628 07:52:31.414455 (+    73us) write_op.cc:276] Timestamp: P: 1751097151414288 usec, L: 0
0628 07:52:31.414471 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:31.414935 (+   464us) log.cc:844] Serialized 913 byte log entry
0628 07:52:31.442634 (+ 27699us) op_driver.cc:464] REPLICATION: finished
0628 07:52:31.443097 (+   463us) write_op.cc:301] APPLY: starting
0628 07:52:31.443148 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:31.443242 (+    94us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:31.443322 (+    80us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:31.443542 (+   220us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:31.443671 (+   129us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:31.443704 (+    33us) write_op.cc:312] APPLY: finished
0628 07:52:31.443911 (+   207us) log.cc:844] Serialized 29 byte log entry
0628 07:52:31.444037 (+   126us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.444095 (+    58us) write_op.cc:454] Released schema lock
0628 07:52:31.444122 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:31.444328 (+   206us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.444350 (+    22us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:31.618213 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:31.618551 (+   338us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:31.618573 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:31.618585 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:31.618848 (+   263us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:31.618886 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:31.618907 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:31.619032 (+   125us) tablet.cc:699] Row locks acquired
0628 07:52:31.619044 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:31.619148 (+   104us) write_op.cc:270] Start()
0628 07:52:31.619226 (+    78us) write_op.cc:276] Timestamp: P: 1751097151619099 usec, L: 0
0628 07:52:31.619242 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:31.619681 (+   439us) log.cc:844] Serialized 356 byte log entry
0628 07:52:31.656874 (+ 37193us) op_driver.cc:464] REPLICATION: finished
0628 07:52:31.657244 (+   370us) write_op.cc:301] APPLY: starting
0628 07:52:31.657296 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:31.657382 (+    86us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:31.657458 (+    76us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:31.657645 (+   187us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:31.657823 (+   178us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:31.657875 (+    52us) write_op.cc:312] APPLY: finished
0628 07:52:31.658037 (+   162us) log.cc:844] Serialized 23 byte log entry
0628 07:52:31.658168 (+   131us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.658240 (+    72us) write_op.cc:454] Released schema lock
0628 07:52:31.658273 (+    33us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:31.669203 (+ 10930us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:31.669227 (+    24us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":69570,"reported_tablets":16,"spinlock_wait_cycles":150964608,"child_traces":[["op",{"apply.queue_time_us":334,"apply.run_cpu_time_us":1206,"apply.run_wall_time_us":1205,"num_ops":2,"prepare.queue_time_us":214082,"prepare.run_cpu_time_us":2031,"prepare.run_wall_time_us":41923,"raft.queue_time_us":1555,"raft.run_cpu_time_us":690,"raft.run_wall_time_us":1493,"replication_time_us":28045}],["op",{"apply.queue_time_us":235,"apply.run_cpu_time_us":1221,"apply.run_wall_time_us":1222,"num_ops":1,"prepare.queue_time_us":160781,"prepare.run_cpu_time_us":1835,"prepare.run_wall_time_us":50967,"raft.queue_time_us":95,"raft.run_cpu_time_us":541,"raft.run_wall_time_us":1264,"replication_time_us":37531}]]}
I20250628 07:52:34.534760  2248 raft_consensus.cc:513] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.536572  2248 leader_election.cc:290] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.536696   986 raft_consensus.cc:2466] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:34.537470   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:34.533143  2257 tablet_replica.cc:331] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:34.541635  2257 raft_consensus.cc:2241] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.545343  2257 raft_consensus.cc:2270] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.547367  2257 ts_tablet_manager.cc:1905] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:34.409382   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a5aaafa5f2c84037b58a9040e19581dd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:34.554129  1234 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:34.554550  1234 ts_tablet_manager.cc:1397] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.061s	user 0.009s	sys 0.004s
W20250628 07:52:34.563738   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 68) took 2379 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.564162   544 rpcz_store.cc:269] 0628 07:52:32.184335 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:32.303343 (+119008us) service_pool.cc:225] Handling call
0628 07:52:32.337058 (+ 33715us) spinlock_profiling.cc:243] Waited 29 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.390226 (+2053168us) spinlock_profiling.cc:243] Waited 16.8 ms on lock 0x7b6800101960. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5bea863af1 00007f5bea84112a 00007f5bea83f868 00007f5bea83de90 00007f5bea9400ae 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc
0628 07:52:34.563668 (+173442us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":147188,"reported_tablets":32,"spinlock_wait_cycles":105380096}
W20250628 07:52:34.571544   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 223684us
I20250628 07:52:34.560706  2262 raft_consensus.cc:491] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.579691  2262 raft_consensus.cc:513] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.584203  2206 ts_tablet_manager.cc:1918] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:34.586529  2262 leader_election.cc:290] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:34.587781  2206 log.cc:1199] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:34.588587  2206 ts_tablet_manager.cc:1939] T 9579dd44e5e241aea2671be28e16ef7c P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:34.589497   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:34.596104   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:34.598657   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:34.598968   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c84bc632fbed42de8d7f5bc303bcaa61" } modify_external_catalogs: true
I20250628 07:52:34.599454   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_c84bc632fbed42de8d7f5bc303bcaa61" } modify_external_catalogs: true
I20250628 07:52:34.599750  2237 ts_tablet_manager.cc:1905] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:34.599714  1062 leader_election.cc:343] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:34.600262  1062 leader_election.cc:343] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.600572  1062 leader_election.cc:304] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.614847  2206 tablet_replica.cc:331] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:34.615697  2206 raft_consensus.cc:2241] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.535444   542 master_service.cc:948] ReplaceTablet: received request to replace tablet fef70db0996f48c69574337f8f7ecf37 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:34.594161   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 9579dd44e5e241aea2671be28e16ef7c (table default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]) successfully deleted
I20250628 07:52:34.617388  2206 raft_consensus.cc:2270] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:34.617684   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 532ba24d0d644313b9ef4e3ec10b0fed because the tablet was not found. No further retry: Not found: Tablet not found: 532ba24d0d644313b9ef4e3ec10b0fed
W20250628 07:52:34.618217   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 0f29c8ffb7df410295278eee9c578b03 because the tablet was not found. No further retry: Not found: Tablet not found: 0f29c8ffb7df410295278eee9c578b03
I20250628 07:52:34.619211  2206 ts_tablet_manager.cc:1905] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:34.620525   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ef060774191146168ee8d1e13c9cd71b because the tablet was not found. No further retry: Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
W20250628 07:52:34.621073   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e28ab08f878d4d4b86a5646f687b08fc because the tablet was not found. No further retry: Not found: Tablet not found: e28ab08f878d4d4b86a5646f687b08fc
I20250628 07:52:34.621207   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_52441cbec305498ebed774e69004a383" } modify_external_catalogs: true
I20250628 07:52:34.638610  2238 tablet_bootstrap.cc:700] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Time spent opening tablet: real 0.154s	user 0.004s	sys 0.000s
W20250628 07:52:34.496788   541 rpcz_store.cc:269] 0628 07:52:31.165574 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:31.229692 (+ 64118us) service_pool.cc:225] Handling call
0628 07:52:34.496296 (+3266604us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:33.517418 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.517826 (+   408us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.517852 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:33.517865 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:33.518112 (+   247us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.518149 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.518171 (+    22us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:33.518302 (+   131us) tablet.cc:699] Row locks acquired
0628 07:52:33.518313 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:33.518421 (+   108us) write_op.cc:270] Start()
0628 07:52:33.518503 (+    82us) write_op.cc:276] Timestamp: P: 1751097153518335 usec, L: 0
0628 07:52:33.518518 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.519043 (+   525us) log.cc:844] Serialized 356 byte log entry
0628 07:52:33.558480 (+ 39437us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.558772 (+   292us) write_op.cc:301] APPLY: starting
0628 07:52:33.558826 (+    54us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.558922 (+    96us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.558999 (+    77us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.559157 (+   158us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.559290 (+   133us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:33.559329 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:33.559488 (+   159us) log.cc:844] Serialized 23 byte log entry
0628 07:52:33.559618 (+   130us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.559678 (+    60us) write_op.cc:454] Released schema lock
0628 07:52:33.559708 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:33.559934 (+   226us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.559957 (+    23us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:33.587552 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.587944 (+   392us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.587965 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:33.587976 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:33.588215 (+   239us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.588253 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.588279 (+    26us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:33.588415 (+   136us) tablet.cc:699] Row locks acquired
0628 07:52:33.588429 (+    14us) write_op.cc:260] PREPARE: finished
0628 07:52:33.588531 (+   102us) write_op.cc:270] Start()
0628 07:52:33.588604 (+    73us) write_op.cc:276] Timestamp: P: 1751097153588491 usec, L: 0
0628 07:52:33.588621 (+    17us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.589057 (+   436us) log.cc:844] Serialized 356 byte log entry
0628 07:52:33.599257 (+ 10200us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.599570 (+   313us) write_op.cc:301] APPLY: starting
0628 07:52:33.599624 (+    54us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.599706 (+    82us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.599802 (+    96us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.599957 (+   155us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.600624 (+   667us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:33.600665 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:33.600835 (+   170us) log.cc:844] Serialized 23 byte log entry
0628 07:52:33.600976 (+   141us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.601035 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:33.601064 (+    29us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:33.601289 (+   225us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.601310 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":80931,"reported_tablets":26,"child_traces":[["op",{"apply.queue_time_us":157,"apply.run_cpu_time_us":1137,"apply.run_wall_time_us":1136,"num_ops":1,"prepare.queue_time_us":6135,"prepare.run_cpu_time_us":1905,"prepare.run_wall_time_us":1905,"raft.queue_time_us":2250,"raft.run_cpu_time_us":679,"raft.run_wall_time_us":679,"replication_time_us":39872}],["op",{"apply.queue_time_us":183,"apply.run_cpu_time_us":1686,"apply.run_wall_time_us":1686,"num_ops":1,"prepare.queue_time_us":93,"prepare.run_cpu_time_us":1798,"prepare.run_wall_time_us":3944,"raft.queue_time_us":74,"raft.run_cpu_time_us":551,"raft.run_wall_time_us":551,"replication_time_us":10554}]]}
I20250628 07:52:34.646931   874 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:34.651819   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 1c65fa809b17418c85d4d72ae1e0ee3a
W20250628 07:52:34.423228   543 rpcz_store.cc:269] 0628 07:52:33.487227 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:33.643611 (+156384us) service_pool.cc:225] Handling call
0628 07:52:33.922492 (+278881us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.922519 (+    27us) write_op.cc:454] Released schema lock
0628 07:52:34.422905 (+500386us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:33.905772 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.906149 (+   377us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.906173 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:33.906189 (+    16us) tablet.cc:660] Decoding operations
0628 07:52:33.906579 (+   390us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.906617 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.906640 (+    23us) tablet.cc:683] Acquiring locks for 3 operations
0628 07:52:33.906875 (+   235us) tablet.cc:699] Row locks acquired
0628 07:52:33.906887 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:33.906995 (+   108us) write_op.cc:270] Start()
0628 07:52:33.907075 (+    80us) write_op.cc:276] Timestamp: P: 1751097153906922 usec, L: 0
0628 07:52:33.907090 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.907562 (+   472us) log.cc:844] Serialized 1293 byte log entry
0628 07:52:33.915963 (+  8401us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.916257 (+   294us) write_op.cc:301] APPLY: starting
0628 07:52:33.916303 (+    46us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.916413 (+   110us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.916506 (+    93us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.916870 (+   364us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.917018 (+   148us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=3
0628 07:52:33.917057 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:33.917248 (+   191us) log.cc:844] Serialized 35 byte log entry
0628 07:52:33.917374 (+   126us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.917440 (+    66us) write_op.cc:454] Released schema lock
0628 07:52:33.917470 (+    30us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:33.957312 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:33.957707 (+   395us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:33.957730 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:33.957742 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:33.957992 (+   250us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:33.958031 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:33.958052 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:33.958182 (+   130us) tablet.cc:699] Row locks acquired
0628 07:52:33.958195 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:33.958294 (+    99us) write_op.cc:270] Start()
0628 07:52:33.958366 (+    72us) write_op.cc:276] Timestamp: P: 1751097153958217 usec, L: 0
0628 07:52:33.958379 (+    13us) op_driver.cc:348] REPLICATION: starting
0628 07:52:33.958857 (+   478us) log.cc:844] Serialized 356 byte log entry
0628 07:52:33.972888 (+ 14031us) op_driver.cc:464] REPLICATION: finished
0628 07:52:33.973574 (+   686us) write_op.cc:301] APPLY: starting
0628 07:52:33.973628 (+    54us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:33.973706 (+    78us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:33.973788 (+    82us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:33.973943 (+   155us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:33.974057 (+   114us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:33.974115 (+    58us) write_op.cc:312] APPLY: finished
0628 07:52:33.974271 (+   156us) log.cc:844] Serialized 23 byte log entry
0628 07:52:33.974401 (+   130us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.974460 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:33.974490 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:33.974704 (+   214us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:33.974727 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":99094,"reported_tablets":33,"child_traces":[["op",{"apply.queue_time_us":155,"apply.run_cpu_time_us":1382,"apply.run_wall_time_us":1382,"num_ops":3,"prepare.queue_time_us":4617,"prepare.run_cpu_time_us":2046,"prepare.run_wall_time_us":2046,"raft.queue_time_us":1173,"raft.run_cpu_time_us":654,"raft.run_wall_time_us":654,"replication_time_us":8780}],["op",{"apply.queue_time_us":526,"apply.run_cpu_time_us":1113,"apply.run_wall_time_us":1112,"num_ops":1,"prepare.queue_time_us":10439,"prepare.run_cpu_time_us":1809,"prepare.run_wall_time_us":1809,"raft.queue_time_us":1181,"raft.run_cpu_time_us":702,"raft.run_wall_time_us":702,"replication_time_us":14425}]]}
I20250628 07:52:34.653903   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.655236   544 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 363) took 2505 ms. Trace:
I20250628 07:52:34.656010  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.657498  2262 raft_consensus.cc:2747] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:34.658200  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 57956us
W20250628 07:52:34.660796   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 128818us
I20250628 07:52:34.657121  2257 ts_tablet_manager.cc:1918] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:34.661902  1234 raft_consensus.cc:357] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.662571  1234 raft_consensus.cc:383] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.662916  1234 raft_consensus.cc:738] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.663101  2257 log.cc:1199] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/b51b51eb951d45a78bd2bffe47834b95
I20250628 07:52:34.663496  1234 consensus_queue.cc:260] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.665822  2257 ts_tablet_manager.cc:1939] T b51b51eb951d45a78bd2bffe47834b95 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:34.666970   794 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:34.669890  1234 ts_tablet_manager.cc:1428] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.115s	user 0.005s	sys 0.001s
I20250628 07:52:34.670151   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet b51b51eb951d45a78bd2bffe47834b95 (table default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]) successfully deleted
I20250628 07:52:34.670863  1234 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:34.670002  2257 tablet_replica.cc:331] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:34.676661  2257 raft_consensus.cc:2241] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.677366  2257 pending_rounds.cc:70] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Trying to abort 1 pending ops.
I20250628 07:52:34.677853  2257 pending_rounds.cc:77] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493937449287680 op_type: NO_OP noop_request { }
I20250628 07:52:34.655512   544 rpcz_store.cc:276] 0628 07:52:32.150160 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.598684 (+2448524us) service_pool.cc:225] Handling call
0628 07:52:34.601423 (+  2739us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:34.603022 (+  1599us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:34.603131 (+   109us) catalog_manager.cc:2799] Locking tablets
0628 07:52:34.603603 (+   472us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:34.651437 (+ 47834us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:34.651632 (+   195us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:34.651729 (+    97us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:34.655186 (+  3457us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.607146 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.607447 (+   301us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.607466 (+    19us) write_op.cc:435] Acquired schema lock
0628 07:52:34.607477 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:34.607759 (+   282us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.607812 (+    53us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.607831 (+    19us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.608058 (+   227us) tablet.cc:699] Row locks acquired
0628 07:52:34.608070 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:34.608157 (+    87us) write_op.cc:270] Start()
0628 07:52:34.608223 (+    66us) write_op.cc:276] Timestamp: P: 1751097154608125 usec, L: 0
0628 07:52:34.608237 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.608618 (+   381us) log.cc:844] Serialized 771 byte log entry
0628 07:52:34.623587 (+ 14969us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.649748 (+ 26161us) write_op.cc:301] APPLY: starting
0628 07:52:34.649801 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.649899 (+    98us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.649980 (+    81us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.650225 (+   245us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.650353 (+   128us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:34.650398 (+    45us) write_op.cc:312] APPLY: finished
0628 07:52:34.650591 (+   193us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.651170 (+   579us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.651229 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:34.651258 (+    29us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":26039,"apply.run_cpu_time_us":1261,"apply.run_wall_time_us":1691,"num_ops":2,"prepare.queue_time_us":2069,"prepare.run_cpu_time_us":1718,"prepare.run_wall_time_us":12718,"raft.queue_time_us":3330,"raft.run_cpu_time_us":8,"raft.run_wall_time_us":9,"replication_time_us":15281,"thread_start_us":49128,"threads_started":1}]]}
I20250628 07:52:34.678527  2257 raft_consensus.cc:2887] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:34.679152  2257 raft_consensus.cc:2270] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.682924   963 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.683307  2257 ts_tablet_manager.cc:1905] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:34.686473   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 69) took 1485 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.688493   546 rpcz_store.cc:269] 0628 07:52:33.201468 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:33.631321 (+429853us) service_pool.cc:225] Handling call
0628 07:52:34.686396 (+1055075us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":196103,"reported_tablets":38}
I20250628 07:52:34.668792  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "08afbaee7cb74ce8a6b2958a6828eca1" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:34.695573  1125 raft_consensus.cc:3058] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:34.700379  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.700472  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.698868   544 master_service.cc:948] ReplaceTablet: received request to replace tablet f845f32699044867af6f0f2b655cad8e from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:34.705325  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.707660   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.712877  2257 ts_tablet_manager.cc:1918] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:34.713515  2257 log.cc:1199] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:34.714207  2257 ts_tablet_manager.cc:1939] T cc0bf0edebb140209531ccf917fd1ed8 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:34.718341   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet cc0bf0edebb140209531ccf917fd1ed8 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
W20250628 07:52:34.662468   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 64418.1us
I20250628 07:52:34.719218  2257 tablet_replica.cc:331] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:34.720979   757 leader_election.cc:343] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.721287  1125 raft_consensus.cc:2466] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
W20250628 07:52:34.718500   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 55) took 956 ms (client timeout 1000 ms). Trace:
W20250628 07:52:34.725251   538 rpcz_store.cc:269] 0628 07:52:33.761716 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:33.765350 (+  3634us) service_pool.cc:225] Handling call
0628 07:52:34.718452 (+953102us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.451133 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.451525 (+   392us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.451561 (+    36us) write_op.cc:435] Acquired schema lock
0628 07:52:34.451571 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:34.451899 (+   328us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.451937 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.451956 (+    19us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.452192 (+   236us) tablet.cc:699] Row locks acquired
0628 07:52:34.452203 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:34.452298 (+    95us) write_op.cc:270] Start()
0628 07:52:34.452373 (+    75us) write_op.cc:276] Timestamp: P: 1751097154452233 usec, L: 0
0628 07:52:34.452388 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.452821 (+   433us) log.cc:844] Serialized 913 byte log entry
0628 07:52:34.460391 (+  7570us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.460656 (+   265us) write_op.cc:301] APPLY: starting
0628 07:52:34.460704 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.460806 (+   102us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.460886 (+    80us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.461160 (+   274us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.461279 (+   119us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:34.461323 (+    44us) write_op.cc:312] APPLY: finished
0628 07:52:34.461470 (+   147us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.461589 (+   119us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.461650 (+    61us) write_op.cc:454] Released schema lock
0628 07:52:34.461680 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.461882 (+   202us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.461904 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":30,"child_traces":[["op",{"apply.queue_time_us":151,"apply.run_cpu_time_us":1191,"apply.run_wall_time_us":1191,"num_ops":2,"prepare.queue_time_us":816,"prepare.run_cpu_time_us":1941,"prepare.run_wall_time_us":1941,"raft.queue_time_us":71,"raft.run_cpu_time_us":600,"raft.run_wall_time_us":19290,"replication_time_us":7932,"spinlock_wait_cycles":35584}]]}
I20250628 07:52:34.736416  2257 raft_consensus.cc:2241] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.736917  2257 raft_consensus.cc:2270] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.740841  2257 ts_tablet_manager.cc:1905] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:34.742269   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:34.750579   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b2d49e5a06384758bce5e7328b6a10f8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:34.752925   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_dddf2413c21f4d4e8c3da5c33cef8a01"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:34.754403   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:34.782696   543 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_62272d797cf643d295305387e17a746b" } new_table_name: "default.Table_3712d707dd104673a98891d70efe27ea" modify_external_catalogs: true
W20250628 07:52:34.786808   757 leader_election.cc:343] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.790262   963 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.790849   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.791381   955 tablet_service.cc:1515] Processing DeleteTablet for tablet ef060774191146168ee8d1e13c9cd71b with delete_type TABLET_DATA_DELETED (replaced by tablet 1fe54aa8c2a04c238afb82ef9ff7fa81) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:34.793291   920 leader_election.cc:343] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:34.793780   920 leader_election.cc:304] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
W20250628 07:52:34.804739   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:34.806564  2233 raft_consensus.cc:2747] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:34.808234  2206 ts_tablet_manager.cc:1918] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:34.808593  2206 log.cc:1199] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/5ec8c1d9386a4922b55df5e06039858d
I20250628 07:52:34.809445  2206 ts_tablet_manager.cc:1939] T 5ec8c1d9386a4922b55df5e06039858d P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:34.812417   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 5ec8c1d9386a4922b55df5e06039858d (table default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]) successfully deleted
I20250628 07:52:34.813077  1863 raft_consensus.cc:491] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.813561  1863 raft_consensus.cc:513] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.810901  2257 ts_tablet_manager.cc:1918] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:34.815371  1863 leader_election.cc:290] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.816207  2257 log.cc:1199] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/83ffa65248fc4db3aecc5f91b648b83f
W20250628 07:52:34.817369   921 leader_election.cc:343] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: b51b51eb951d45a78bd2bffe47834b95
I20250628 07:52:34.818228  2206 tablet_replica.cc:331] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:34.819144  2206 raft_consensus.cc:2241] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:34.820122  2206 raft_consensus.cc:2270] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.820972   540 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 359) took 3057 ms. Trace:
I20250628 07:52:34.821224  2257 ts_tablet_manager.cc:1939] T 83ffa65248fc4db3aecc5f91b648b83f P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:34.821405   540 rpcz_store.cc:276] 0628 07:52:31.763484 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.397760 (+2634276us) service_pool.cc:225] Handling call
0628 07:52:34.400025 (+  2265us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:34.600271 (+200246us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:34.666396 (+ 66125us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:34.820611 (+154215us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:34.820913 (+   302us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.605271 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.605639 (+   368us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.605666 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:34.605679 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:34.605954 (+   275us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.605990 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.606010 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.606208 (+   198us) tablet.cc:699] Row locks acquired
0628 07:52:34.606220 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:34.606329 (+   109us) write_op.cc:270] Start()
0628 07:52:34.606398 (+    69us) write_op.cc:276] Timestamp: P: 1751097154606282 usec, L: 0
0628 07:52:34.606413 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.606839 (+   426us) log.cc:844] Serialized 433 byte log entry
0628 07:52:34.623378 (+ 16539us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.624038 (+   660us) write_op.cc:301] APPLY: starting
0628 07:52:34.624088 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.624179 (+    91us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.624251 (+    72us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.624623 (+   372us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.624748 (+   125us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:34.624799 (+    51us) write_op.cc:312] APPLY: finished
0628 07:52:34.624956 (+   157us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.625115 (+   159us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.625174 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:34.625201 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.649483 (+ 24282us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.649509 (+    26us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":530,"apply.run_cpu_time_us":5289,"apply.run_wall_time_us":25391,"num_ops":2,"prepare.queue_time_us":3874,"prepare.run_cpu_time_us":1793,"prepare.run_wall_time_us":1793,"raft.queue_time_us":4214,"raft.run_cpu_time_us":755,"raft.run_wall_time_us":755,"replication_time_us":16882}]]}
I20250628 07:52:34.822669  1229 tablet_bootstrap.cc:654] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:34.825934   757 leader_election.cc:343] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.833884  2229 consensus_queue.cc:1035] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.009s
I20250628 07:52:34.845750   539 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 360) took 3063 ms. Trace:
I20250628 07:52:34.851779  2229 raft_consensus.cc:491] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.849196   539 rpcz_store.cc:276] 0628 07:52:31.782613 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.407668 (+2625055us) service_pool.cc:225] Handling call
0628 07:52:34.561285 (+153617us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:34.670837 (+109552us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:34.737586 (+ 66749us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:34.845389 (+107803us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:34.845690 (+   301us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.700178 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.705494 (+  5316us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.705562 (+    68us) write_op.cc:435] Acquired schema lock
0628 07:52:34.705577 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:34.705895 (+   318us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.705952 (+    57us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.705999 (+    47us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.706239 (+   240us) tablet.cc:699] Row locks acquired
0628 07:52:34.706257 (+    18us) write_op.cc:260] PREPARE: finished
0628 07:52:34.706377 (+   120us) write_op.cc:270] Start()
0628 07:52:34.706466 (+    89us) write_op.cc:276] Timestamp: P: 1751097154706322 usec, L: 0
0628 07:52:34.706486 (+    20us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.707049 (+   563us) log.cc:844] Serialized 433 byte log entry
0628 07:52:34.718703 (+ 11654us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.718990 (+   287us) write_op.cc:301] APPLY: starting
0628 07:52:34.719045 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.719178 (+   133us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.719275 (+    97us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.719711 (+   436us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.719872 (+   161us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:34.719922 (+    50us) write_op.cc:312] APPLY: finished
0628 07:52:34.720102 (+   180us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.720242 (+   140us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.737293 (+ 17051us) spinlock_profiling.cc:243] Waited 1.88 ms on lock 0x7b0c00086490. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8f2dd11 00007f5be906e940 00007f5be906f210 00007f5be906f08e 00007f5be90109d0 00007f5be900fbd8 00007f5be900f5dc 00007f5be8ffb619 00007f5be8ffade3 00007f5be8ffeff1 00007f5be8ffefa9
0628 07:52:34.737348 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:34.737384 (+    36us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.738032 (+   648us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.738084 (+    52us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":155,"apply.run_cpu_time_us":6872,"apply.run_wall_time_us":18940,"num_ops":2,"prepare.queue_time_us":28219,"prepare.run_cpu_time_us":2358,"prepare.run_wall_time_us":15906,"raft.queue_time_us":98,"raft.run_cpu_time_us":1380,"raft.run_wall_time_us":1503,"replication_time_us":12113,"spinlock_wait_cycles":4313344}]]}
I20250628 07:52:34.852165  2229 raft_consensus.cc:513] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.866720   544 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 366) took 2460 ms. Trace:
I20250628 07:52:34.867118  2229 leader_election.cc:290] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.867100   544 rpcz_store.cc:276] 0628 07:52:32.406434 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.698745 (+2292311us) service_pool.cc:225] Handling call
0628 07:52:34.866661 (+167916us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:34.839128  2206 ts_tablet_manager.cc:1905] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:34.868913   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:34.869997  2270 tablet.cc:1722] T 661d92febf1142f99815f70ae9b27edc P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:34.870980   955 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:34.871173   542 catalog_manager.cc:6652] ReplaceTablet: tablet fef70db0996f48c69574337f8f7ecf37 deleted and replaced by tablet d3e486f93a67441fa2da3a72af327503
I20250628 07:52:34.871476   542 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 362) took 2794 ms. Trace:
I20250628 07:52:34.871941  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.873745   794 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:34.874331   757 consensus_peers.cc:489] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:34.876178   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_76b6c7d9f5574825bcc78b251dac8721" } new_table_name: "default.Table_06835b5c509a43aca747e21ac8381c29" modify_external_catalogs: true
I20250628 07:52:34.877705   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:34.878634  1230 thread.cc:640] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.595s	user 0.026s	sys 0.096s
W20250628 07:52:34.878937  1230 thread.cc:606] log-alloc [worker] (thread pool) Time spent starting thread: real 0.595s	user 0.026s	sys 0.096s
W20250628 07:52:34.880106   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 73901.1us
I20250628 07:52:34.882261   538 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 367) took 2377 ms. Trace:
I20250628 07:52:34.871758   542 rpcz_store.cc:276] 0628 07:52:32.076845 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.535264 (+2458419us) service_pool.cc:225] Handling call
0628 07:52:34.871429 (+336165us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.649670 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.650060 (+   390us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.650083 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:34.650096 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:34.650388 (+   292us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.650421 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.650443 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.650646 (+   203us) tablet.cc:699] Row locks acquired
0628 07:52:34.650658 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:34.650757 (+    99us) write_op.cc:270] Start()
0628 07:52:34.650839 (+    82us) write_op.cc:276] Timestamp: P: 1751097154650717 usec, L: 0
0628 07:52:34.650854 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.683484 (+ 32630us) log.cc:844] Serialized 628 byte log entry
0628 07:52:34.686780 (+  3296us) spinlock_profiling.cc:243] Waited 32.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:34.696137 (+  9357us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.696605 (+   468us) write_op.cc:301] APPLY: starting
0628 07:52:34.696656 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.696757 (+   101us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.696792 (+    35us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.697116 (+   324us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.697339 (+   223us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:34.697395 (+    56us) write_op.cc:312] APPLY: finished
0628 07:52:34.697596 (+   201us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.697742 (+   146us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.697803 (+    61us) write_op.cc:454] Released schema lock
0628 07:52:34.697842 (+    39us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.699881 (+  2039us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.699906 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":292,"apply.run_cpu_time_us":1446,"apply.run_wall_time_us":1446,"num_ops":2,"prepare.queue_time_us":91,"prepare.run_cpu_time_us":4140,"prepare.run_wall_time_us":50196,"raft.queue_time_us":93,"raft.run_cpu_time_us":551,"raft.run_wall_time_us":551,"replication_time_us":45200,"spinlock_wait_cycles":74014592}]]}
I20250628 07:52:34.895069  2246 raft_consensus.cc:491] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.882905   538 rpcz_store.cc:276] 0628 07:52:32.505242 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.738778 (+2233536us) service_pool.cc:225] Handling call
0628 07:52:34.752839 (+ 14061us) spinlock_profiling.cc:243] Waited 13.3 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea940d36 00007f5be865b6a4 00007f5be865b632 00007f5be865b561 00007f5be865b4dc 00007f5be865a792 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.852956 (+100117us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:34.854591 (+  1635us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:34.881720 (+ 27129us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:34.881982 (+   262us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:34.882209 (+   227us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.856580 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.856841 (+   261us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.856864 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:34.856881 (+    17us) tablet.cc:660] Decoding operations
0628 07:52:34.857154 (+   273us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.857193 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.857215 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.857399 (+   184us) tablet.cc:699] Row locks acquired
0628 07:52:34.857411 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:34.857508 (+    97us) write_op.cc:270] Start()
0628 07:52:34.857603 (+    95us) write_op.cc:276] Timestamp: P: 1751097154857448 usec, L: 0
0628 07:52:34.857618 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.858087 (+   469us) log.cc:844] Serialized 433 byte log entry
0628 07:52:34.874297 (+ 16210us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.875222 (+   925us) write_op.cc:301] APPLY: starting
0628 07:52:34.875281 (+    59us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.875374 (+    93us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.875454 (+    80us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.875860 (+   406us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.875994 (+   134us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:34.876037 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:34.876338 (+   301us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.876481 (+   143us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.876543 (+    62us) write_op.cc:454] Released schema lock
0628 07:52:34.876577 (+    34us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.876795 (+   218us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.876817 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":30671232,"child_traces":[["op",{"apply.queue_time_us":778,"apply.run_cpu_time_us":1422,"apply.run_wall_time_us":1526,"num_ops":2,"prepare.queue_time_us":818,"prepare.run_cpu_time_us":1771,"prepare.run_wall_time_us":3361,"raft.queue_time_us":76,"raft.run_cpu_time_us":580,"raft.run_wall_time_us":580,"replication_time_us":16593}]]}
W20250628 07:52:34.897192   757 leader_election.cc:343] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.901219  2229 consensus_queue.cc:1035] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.905687   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4eb55153d17a4680b1de4c5ecba431d2" } modify_external_catalogs: true
I20250628 07:52:34.906194   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4eb55153d17a4680b1de4c5ecba431d2" } modify_external_catalogs: true
W20250628 07:52:34.909863   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 2f8099bd0c9649b492edd8b657fd19e3 consensus state has no leader
I20250628 07:52:34.910034  2147 consensus_queue.cc:1035] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:34.910967   758 leader_election.cc:304] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.911630  2231 raft_consensus.cc:2802] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:34.912048  2231 raft_consensus.cc:695] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
W20250628 07:52:34.912140   758 consensus_peers.cc:489] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cc0bf0edebb140209531ccf917fd1ed8. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:34.912793  2231 consensus_queue.cc:237] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.926406  1230 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:34.927112  1230 ts_tablet_manager.cc:1397] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.684s	user 0.036s	sys 0.096s
I20250628 07:52:34.927663  2273 raft_consensus.cc:491] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.928277  2273 raft_consensus.cc:513] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:34.932577   921 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:34.933503  2273 leader_election.cc:290] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.937357  2262 raft_consensus.cc:491] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.937302  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:34.937863  1126 raft_consensus.cc:2466] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:34.937758  2262 raft_consensus.cc:513] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.939461  2262 leader_election.cc:290] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:34.940737  2231 raft_consensus.cc:491] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:34.941152  2231 raft_consensus.cc:513] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.943440  2231 leader_election.cc:290] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:34.944458   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:34.947367   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 45132.8us
W20250628 07:52:34.948482  1062 leader_election.cc:343] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:34.951673   541 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 364) took 2680 ms. Trace:
I20250628 07:52:34.952282   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 83ffa65248fc4db3aecc5f91b648b83f (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:34.899286  1234 tablet_bootstrap.cc:654] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:34.952009   541 rpcz_store.cc:276] 0628 07:52:32.271709 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.644427 (+2372718us) service_pool.cc:225] Handling call
0628 07:52:34.742120 (+ 97693us) spinlock_profiling.cc:243] Waited 38.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea940d36 00007f5be865b6a4 00007f5be865b632 00007f5be865b561 00007f5be865b4dc 00007f5be865a792 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.873623 (+131503us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:34.874972 (+  1349us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:34.924331 (+ 49359us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:34.951332 (+ 27001us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:34.951608 (+   276us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.876109 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.876922 (+   813us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.876946 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:34.876958 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:34.877232 (+   274us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.877265 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.877286 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.885178 (+  7892us) tablet.cc:699] Row locks acquired
0628 07:52:34.885194 (+    16us) write_op.cc:260] PREPARE: finished
0628 07:52:34.885297 (+   103us) write_op.cc:270] Start()
0628 07:52:34.885373 (+    76us) write_op.cc:276] Timestamp: P: 1751097154885231 usec, L: 0
0628 07:52:34.885389 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.885818 (+   429us) log.cc:844] Serialized 433 byte log entry
0628 07:52:34.907940 (+ 22122us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.908348 (+   408us) write_op.cc:301] APPLY: starting
0628 07:52:34.908398 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.908489 (+    91us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.908585 (+    96us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.908963 (+   378us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.909136 (+   173us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:34.909181 (+    45us) write_op.cc:312] APPLY: finished
0628 07:52:34.909341 (+   160us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.909490 (+   149us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.909556 (+    66us) write_op.cc:454] Released schema lock
0628 07:52:34.909584 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.926104 (+ 16520us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.926127 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":87804288,"child_traces":[["op",{"apply.queue_time_us":273,"apply.run_cpu_time_us":1412,"apply.run_wall_time_us":1413,"num_ops":2,"prepare.queue_time_us":88,"prepare.run_cpu_time_us":1899,"prepare.run_wall_time_us":49978,"raft.queue_time_us":98,"raft.run_cpu_time_us":639,"raft.run_wall_time_us":645,"replication_time_us":22470}]]}
I20250628 07:52:34.958730   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet d3501090b0a34ca9882dde55b19e3621
I20250628 07:52:34.961325  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet d3501090b0a34ca9882dde55b19e3621 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:34.961694  2257 tablet_replica.cc:331] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:34.962587  2257 raft_consensus.cc:2241] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:34.963030  2257 raft_consensus.cc:2270] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:34.966830  2238 log.cc:826] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Log is configured to *not* fsync() on all Append() calls
I20250628 07:52:34.968644   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:34.972510  2206 ts_tablet_manager.cc:1918] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:34.962988  1230 raft_consensus.cc:357] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:34.974529  1230 raft_consensus.cc:383] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:34.974819  1230 raft_consensus.cc:738] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:34.975404  1230 consensus_queue.cc:260] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:34.976646   757 leader_election.cc:343] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:34.994622   758 consensus_peers.cc:489] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 83ffa65248fc4db3aecc5f91b648b83f. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:34.995963  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:34.996513  1126 raft_consensus.cc:2466] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:35.026217   758 leader_election.cc:304] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:34.895565  2246 raft_consensus.cc:513] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.029645   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_dea57d0c6e4149b1a063b0ec3b5bc013"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.030473  2257 ts_tablet_manager.cc:1905] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:35.031167   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_dea57d0c6e4149b1a063b0ec3b5bc013 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.005648   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b51b51eb951d45a78bd2bffe47834b95" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:35.038599  2279 tablet.cc:1722] T 661d92febf1142f99815f70ae9b27edc P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:35.040285   758 leader_election.cc:304] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.035892   794 tablet_service.cc:1515] Processing DeleteTablet for tablet d3501090b0a34ca9882dde55b19e3621 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:35.042979  1230 ts_tablet_manager.cc:1428] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.115s	user 0.002s	sys 0.005s
I20250628 07:52:35.044003  1230 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:35.036522  2248 raft_consensus.cc:2802] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:35.027180  2180 consensus_queue.cc:1035] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.013s
I20250628 07:52:35.056509   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_786e13bb34124bc0bb4f5dae8326f725"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.058126   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_786e13bb34124bc0bb4f5dae8326f725 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:34.973734   955 tablet_service.cc:1515] Processing DeleteTablet for tablet d3501090b0a34ca9882dde55b19e3621 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:35.071806  2162 raft_consensus.cc:491] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:35.072232  2162 raft_consensus.cc:513] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.009797  2282 tablet.cc:1722] T 661d92febf1142f99815f70ae9b27edc P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
W20250628 07:52:35.032613   920 leader_election.cc:343] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:35.073846  1230 tablet_bootstrap.cc:654] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.073448   920 leader_election.cc:304] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:35.073992  2162 leader_election.cc:290] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:35.074941  2233 raft_consensus.cc:2747] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:35.074924   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:35.076448  1062 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.055994  2025 raft_consensus.cc:2802] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:35.080145  2206 log.cc:1199] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:35.086113  2206 ts_tablet_manager.cc:1939] T 5475a89692eb450d82cde521ec8c9a73 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.056326  2248 raft_consensus.cc:491] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:35.056203   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:35.089270   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 48020.3us
W20250628 07:52:35.092940  1062 leader_election.cc:343] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.093333  1062 leader_election.cc:304] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.098505  2162 raft_consensus.cc:2747] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:35.101141   920 leader_election.cc:304] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:35.102150  2273 raft_consensus.cc:2747] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:35.103555   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:35.088243  2025 raft_consensus.cc:491] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20250628 07:52:35.105903  1062 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.106212  1062 leader_election.cc:304] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.106626   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b46689c124f54809961340f6c6b317d5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.107323  2162 raft_consensus.cc:2747] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:35.107944   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b46689c124f54809961340f6c6b317d5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.089747  2248 raft_consensus.cc:3058] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.115079  1230 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:35.115550  1230 ts_tablet_manager.cc:1397] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.072s	user 0.007s	sys 0.002s
I20250628 07:52:35.117820   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 5475a89692eb450d82cde521ec8c9a73 (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]) successfully deleted
I20250628 07:52:35.119467  2206 tablet_replica.cc:331] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.119848  1229 tablet_bootstrap.cc:492] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.120457  2206 raft_consensus.cc:2241] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.120858  2206 raft_consensus.cc:2270] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.120301  1229 ts_tablet_manager.cc:1397] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.764s	user 0.028s	sys 0.007s
I20250628 07:52:35.122628  2206 ts_tablet_manager.cc:1905] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.123600  1229 raft_consensus.cc:357] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.124228  1229 raft_consensus.cc:383] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.124106   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:35.125165  1229 raft_consensus.cc:738] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.127735  1229 consensus_queue.cc:260] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:35.128654   757 leader_election.cc:343] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.115657  2025 raft_consensus.cc:3058] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.133920  1229 ts_tablet_manager.cc:1428] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.012s	user 0.003s	sys 0.001s
I20250628 07:52:35.134488  1230 raft_consensus.cc:357] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.135138  1230 raft_consensus.cc:383] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.135365  1230 raft_consensus.cc:738] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.135882  1230 consensus_queue.cc:260] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.139262  2162 raft_consensus.cc:491] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:35.139647  2162 raft_consensus.cc:513] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.140733   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:35.141212  2162 leader_election.cc:290] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:35.141680  1229 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.142675   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:35.143790  1062 leader_election.cc:343] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.158360   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_a90d3fd61b424016a14e3add816d2218"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.158846  1229 tablet_bootstrap.cc:654] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:35.159780   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a90d3fd61b424016a14e3add816d2218 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.163784   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7e69244decba48b8b9a9f406ccf40827" } modify_external_catalogs: true
I20250628 07:52:35.164337   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_7e69244decba48b8b9a9f406ccf40827" } modify_external_catalogs: true
I20250628 07:52:35.170965  1176 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
W20250628 07:52:35.190917  2238 log.cc:927] Time spent T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Append to log took a long time: real 0.105s	user 0.000s	sys 0.000s
I20250628 07:52:35.200320   542 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:35.200560  1229 tablet_bootstrap.cc:492] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.201810   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:35.206240   794 tablet_service.cc:1515] Processing DeleteTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:35.206566  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:35.200980  1229 ts_tablet_manager.cc:1397] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.059s	user 0.009s	sys 0.001s
I20250628 07:52:35.212250  1229 raft_consensus.cc:357] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.213987  2206 ts_tablet_manager.cc:1918] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:35.214802  2206 log.cc:1199] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/cb62cf19aaf04444845358cc8ad611fb
I20250628 07:52:35.159965  2246 leader_election.cc:290] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.220036  2206 ts_tablet_manager.cc:1939] T cb62cf19aaf04444845358cc8ad611fb P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.220345  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:35.220963  1124 raft_consensus.cc:2466] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:35.221936  2237 ts_tablet_manager.cc:1918] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:35.222281  2237 log.cc:1199] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:35.222648   758 leader_election.cc:304] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.222824  2237 ts_tablet_manager.cc:1939] T b2d49e5a06384758bce5e7328b6a10f8 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:35.223932   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet b2d49e5a06384758bce5e7328b6a10f8 (table default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]) successfully deleted
I20250628 07:52:35.225229  2237 tablet_replica.cc:331] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:35.225329  2147 raft_consensus.cc:2802] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:35.225639  2147 raft_consensus.cc:491] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:35.225744  1229 raft_consensus.cc:383] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.225909  2147 raft_consensus.cc:3058] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.226047  1229 raft_consensus.cc:738] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.236517   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet cb62cf19aaf04444845358cc8ad611fb (table default.Table_92381327fd4e479298ecf33ccdd4eccb [id=474c781f37cc40fb8c9430e27c845d05]) successfully deleted
I20250628 07:52:35.237286  1229 consensus_queue.cc:260] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.237944  2025 raft_consensus.cc:513] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.238299  2206 tablet_replica.cc:331] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.239375  2206 raft_consensus.cc:2241] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:35.247263  2206 raft_consensus.cc:2270] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.249289  2206 ts_tablet_manager.cc:1905] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.243978  2147 raft_consensus.cc:513] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:35.251587   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 125632us
I20250628 07:52:35.246419  1229 ts_tablet_manager.cc:1428] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.037s	user 0.005s	sys 0.000s
I20250628 07:52:35.249655  2248 raft_consensus.cc:513] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.258368   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:35.259924  1062 leader_election.cc:343] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.260272  1062 leader_election.cc:304] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.261960  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:35.262426  1124 raft_consensus.cc:3058] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.265203  2162 raft_consensus.cc:2747] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:35.275296   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f94575e525fe42959d7f99555b557ead"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.276155   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "aea9552ebd5949c8b0b5f23e132e9b6c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
W20250628 07:52:35.277036   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f94575e525fe42959d7f99555b557ead in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.277410  2237 raft_consensus.cc:2241] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.277920  2237 raft_consensus.cc:2270] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.280385  2237 ts_tablet_manager.cc:1905] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.282245  1229 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.286038   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_21bfb5f55d7a4676a0054415d0b5acb6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.287338   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_21bfb5f55d7a4676a0054415d0b5acb6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.298594  1229 tablet_bootstrap.cc:654] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.298594  1124 raft_consensus.cc:2466] T aea9552ebd5949c8b0b5f23e132e9b6c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:35.305362   758 leader_election.cc:304] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:35.306020  2175 raft_consensus.cc:2802] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:35.306407  2175 raft_consensus.cc:695] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:35.307008  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:35.307085  2175 consensus_queue.cc:237] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.305608  2025 leader_election.cc:290] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.307646  1124 raft_consensus.cc:3058] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.258852  2248 leader_election.cc:290] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.323056  2180 consensus_queue.cc:1035] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.324673   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
W20250628 07:52:35.321631   757 leader_election.cc:343] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.327014   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "39ba8057357241b99882e883f6af93e7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:35.328088  1234 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:35.328143   757 leader_election.cc:343] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:35.328471  1234 ts_tablet_manager.cc:1397] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.658s	user 0.005s	sys 0.013s
I20250628 07:52:35.333889  1230 ts_tablet_manager.cc:1428] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.218s	user 0.006s	sys 0.000s
I20250628 07:52:35.334693  1230 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:35.341337   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:35.342620   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:35.343107   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_76b6c7d9f5574825bcc78b251dac8721 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 1 got 0
I20250628 07:52:35.343775   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:35.344523   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:35.340814  1124 raft_consensus.cc:2466] T 39ba8057357241b99882e883f6af93e7 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:35.348023   758 leader_election.cc:304] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:35.361785  1229 tablet_bootstrap.cc:492] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.362195  1229 ts_tablet_manager.cc:1397] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.080s	user 0.010s	sys 0.000s
I20250628 07:52:35.364362  1229 raft_consensus.cc:357] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.366452  2180 raft_consensus.cc:2802] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:35.366859  2180 raft_consensus.cc:695] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:35.367561  2180 consensus_queue.cc:237] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.368464   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7631c6a0720642949d4b63e271a92e4c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.369871   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7631c6a0720642949d4b63e271a92e4c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:35.377218  1928 thread.cc:640] wal-append [worker] (thread pool) Time spent creating pthread: real 0.712s	user 0.016s	sys 0.039s
W20250628 07:52:35.377517  1928 thread.cc:606] wal-append [worker] (thread pool) Time spent starting thread: real 0.712s	user 0.017s	sys 0.039s
W20250628 07:52:35.378434   758 consensus_peers.cc:489] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b51b51eb951d45a78bd2bffe47834b95. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:35.384189  2246 consensus_queue.cc:1035] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.392310  2147 leader_election.cc:290] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.393365   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:35.396778   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_481f2aef57d748fa98e48e26b106b1f5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.364989  1229 raft_consensus.cc:383] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:35.398159   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_481f2aef57d748fa98e48e26b106b1f5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.398962  1229 raft_consensus.cc:738] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.401535  1229 consensus_queue.cc:260] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.402396   981 raft_consensus.cc:3058] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.406451   544 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8146958ca541498babe752911c4346bd" } modify_external_catalogs: true
I20250628 07:52:35.409891  1229 ts_tablet_manager.cc:1428] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.047s	user 0.006s	sys 0.000s
I20250628 07:52:35.413054  1229 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.417865  2206 ts_tablet_manager.cc:1918] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:35.419275  1234 raft_consensus.cc:357] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.419917  1234 raft_consensus.cc:383] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.420172  1234 raft_consensus.cc:738] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.420727  2206 log.cc:1199] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/1c65fa809b17418c85d4d72ae1e0ee3a
I20250628 07:52:35.421725  2206 ts_tablet_manager.cc:1939] T 1c65fa809b17418c85d4d72ae1e0ee3a P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.429230   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]) successfully deleted
I20250628 07:52:35.429257   981 raft_consensus.cc:2466] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:35.429852  2206 tablet_replica.cc:331] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.434952  1229 tablet_bootstrap.cc:654] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.450671  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2f8099bd0c9649b492edd8b657fd19e3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
W20250628 07:52:35.452576   539 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 79d15e600b8b460fab54eb21193f1f40 consensus state has no leader
I20250628 07:52:35.452827   757 leader_election.cc:304] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:35.451177  1124 raft_consensus.cc:3058] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:35.456737   543 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet af13b56df157443c994f747e52b0f458 consensus state has no leader
I20250628 07:52:35.468870  1124 raft_consensus.cc:2466] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:35.480432  2206 raft_consensus.cc:2241] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.481154  2206 raft_consensus.cc:2270] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.483078  2206 ts_tablet_manager.cc:1905] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.487960  1230 tablet_bootstrap.cc:654] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.501488  1229 tablet_bootstrap.cc:492] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.501897  1229 ts_tablet_manager.cc:1397] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.089s	user 0.009s	sys 0.000s
I20250628 07:52:35.502488   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:35.504034   540 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:35.505489   539 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 368) took 1784 ms. Trace:
I20250628 07:52:35.503964  1229 raft_consensus.cc:357] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.506124  1229 raft_consensus.cc:383] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.506381  1229 raft_consensus.cc:738] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.505780   539 rpcz_store.cc:276] 0628 07:52:33.720881 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.852708 (+1131827us) service_pool.cc:225] Handling call
0628 07:52:34.876732 (+ 24024us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:34.918206 (+ 41474us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:34.949622 (+ 31416us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:34.949819 (+   197us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:34.987997 (+ 38178us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:35.505432 (+517435us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.970330 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.970765 (+   435us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.970788 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:34.970801 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:34.971042 (+   241us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.971083 (+    41us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.971108 (+    25us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:34.971251 (+   143us) tablet.cc:699] Row locks acquired
0628 07:52:34.971263 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:34.971365 (+   102us) write_op.cc:270] Start()
0628 07:52:34.971438 (+    73us) write_op.cc:276] Timestamp: P: 1751097154971291 usec, L: 0
0628 07:52:34.971453 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.971895 (+   442us) log.cc:844] Serialized 405 byte log entry
0628 07:52:34.985821 (+ 13926us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.986167 (+   346us) write_op.cc:301] APPLY: starting
0628 07:52:34.986240 (+    73us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.986372 (+   132us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.986479 (+   107us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.986680 (+   201us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.986866 (+   186us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:34.986927 (+    61us) write_op.cc:312] APPLY: finished
0628 07:52:34.987120 (+   193us) log.cc:844] Serialized 23 byte log entry
0628 07:52:34.987291 (+   171us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.987367 (+    76us) write_op.cc:454] Released schema lock
0628 07:52:34.987406 (+    39us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.987671 (+   265us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.987704 (+    33us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":183,"apply.run_cpu_time_us":1453,"apply.run_wall_time_us":1452,"num_ops":1,"prepare.queue_time_us":19336,"prepare.run_cpu_time_us":1827,"prepare.run_wall_time_us":1828,"raft.queue_time_us":640,"raft.run_cpu_time_us":617,"raft.run_wall_time_us":616,"replication_time_us":14274}]]}
I20250628 07:52:35.507977  1229 consensus_queue.cc:260] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.509496   543 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 365) took 3122 ms. Trace:
I20250628 07:52:35.513208   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:35.513685  1229 ts_tablet_manager.cc:1428] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.000s
I20250628 07:52:35.514631   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:35.514794   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 49466.6us
I20250628 07:52:35.515169   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:35.516094   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 63335.6us
I20250628 07:52:35.523489  2025 raft_consensus.cc:2802] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:35.523975  2025 raft_consensus.cc:695] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:35.524699  2025 consensus_queue.cc:237] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.526374  1229 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:35.529649   758 consensus_peers.cc:489] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:35.535377   544 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8146958ca541498babe752911c4346bd" } modify_external_catalogs: true
I20250628 07:52:35.540997  1229 tablet_bootstrap.cc:654] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.545851  2206 ts_tablet_manager.cc:1918] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:35.554862  1863 raft_consensus.cc:491] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:35.557272  1863 raft_consensus.cc:513] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.558746  1863 leader_election.cc:290] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.559859  2206 log.cc:1199] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:35.560111  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:35.560576  2206 ts_tablet_manager.cc:1939] T b2d49e5a06384758bce5e7328b6a10f8 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
W20250628 07:52:35.561480   921 leader_election.cc:343] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:35.562119   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "46ff641001cd4b01a4c09c0d8b79ebf3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:35.563373   920 leader_election.cc:304] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:35.564245  1863 raft_consensus.cc:2747] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:35.566605   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8dcb758f32dc4345acc05a8e494ea755"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.568148   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8dcb758f32dc4345acc05a8e494ea755 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.509806   543 rpcz_store.cc:276] 0628 07:52:32.387102 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.654580 (+2267478us) service_pool.cc:225] Handling call
0628 07:52:34.782578 (+127998us) spinlock_profiling.cc:243] Waited 126 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea9413b6 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc 00007f5be6957206
0628 07:52:34.921137 (+138559us) spinlock_profiling.cc:243] Waited 3.89 ms on lock 0x7b5400002490. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be1a869b1 00007f5be1c069b0 00007f5bea82da7d 00007f5bea825f6b 00007f5bea9413f2 00007f5be8664ca4 00007f5be8664c32 00007f5be8664b61 00007f5be8664adc 00007f5be8663d92 00007f5be6957dcc
0628 07:52:34.921202 (+    65us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:34.956630 (+ 35428us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:34.957190 (+   560us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:34.957439 (+   249us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:34.997524 (+ 40085us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:35.509439 (+511915us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.972238 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.972573 (+   335us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.972593 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:34.972604 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:34.972847 (+   243us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.972880 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.972901 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:34.973024 (+   123us) tablet.cc:699] Row locks acquired
0628 07:52:34.973035 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:34.973142 (+   107us) write_op.cc:270] Start()
0628 07:52:34.973230 (+    88us) write_op.cc:276] Timestamp: P: 1751097154973072 usec, L: 0
0628 07:52:34.973245 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.973665 (+   420us) log.cc:844] Serialized 405 byte log entry
0628 07:52:34.995555 (+ 21890us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.996490 (+   935us) write_op.cc:301] APPLY: starting
0628 07:52:34.996539 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.996622 (+    83us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.996709 (+    87us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.996848 (+   139us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.996954 (+   106us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:34.997006 (+    52us) write_op.cc:312] APPLY: finished
0628 07:52:34.997173 (+   167us) log.cc:844] Serialized 23 byte log entry
0628 07:52:34.997287 (+   114us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.997338 (+    51us) write_op.cc:454] Released schema lock
0628 07:52:34.997362 (+    24us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.997554 (+   192us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.997575 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":299356928,"child_traces":[["op",{"apply.queue_time_us":803,"apply.run_cpu_time_us":1035,"apply.run_wall_time_us":1036,"num_ops":1,"prepare.queue_time_us":13850,"prepare.run_cpu_time_us":1579,"prepare.run_wall_time_us":1579,"replication_time_us":22223}]]}
I20250628 07:52:35.534936   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1cf78a5cb3c5490bae2317600c643635"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.572373   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet b2d49e5a06384758bce5e7328b6a10f8 (table default.Table_5a2877b9c29b49c18d0909b99b08fa80 [id=d91d58d175ad4a07a9cdba1a3cb0888e]) successfully deleted
I20250628 07:52:35.572703   981 raft_consensus.cc:3058] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:35.572801   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1cf78a5cb3c5490bae2317600c643635 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.568652  1234 consensus_queue.cc:260] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.578119  1234 ts_tablet_manager.cc:1428] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.249s	user 0.006s	sys 0.000s
I20250628 07:52:35.578051   982 raft_consensus.cc:3058] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.595407  2025 consensus_queue.cc:1035] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.597612  2206 tablet_replica.cc:331] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.599437  2206 raft_consensus.cc:2241] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.605232  1234 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:35.610785  2206 raft_consensus.cc:2270] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.613035  2206 ts_tablet_manager.cc:1905] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.623252  2237 ts_tablet_manager.cc:1918] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:35.614809  1229 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.623607  2237 log.cc:1199] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:35.623914  1229 ts_tablet_manager.cc:1397] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.098s	user 0.009s	sys 0.000s
I20250628 07:52:35.624222  2237 ts_tablet_manager.cc:1939] T 5475a89692eb450d82cde521ec8c9a73 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:35.626219  1229 raft_consensus.cc:357] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.626868  1229 raft_consensus.cc:383] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.627125  1229 raft_consensus.cc:738] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.633476  1229 consensus_queue.cc:260] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.638406  2257 ts_tablet_manager.cc:1918] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:35.638748  2257 log.cc:1199] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:35.639304  1229 ts_tablet_manager.cc:1428] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
I20250628 07:52:35.640228  1229 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.634105   982 raft_consensus.cc:3058] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:35.655376   540 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 56) took 856 ms (client timeout 1000 ms). Trace:
I20250628 07:52:35.655371   962 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:35.655694   540 rpcz_store.cc:269] 0628 07:52:34.799489 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.824697 (+ 25208us) service_pool.cc:225] Handling call
0628 07:52:35.655317 (+830620us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":223,"reported_tablets":14}
I20250628 07:52:35.661031  2206 ts_tablet_manager.cc:1918] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:35.661356  2206 log.cc:1199] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:35.662088  2206 ts_tablet_manager.cc:1939] T 0dffb3eb778946869b914e5b05a67a69 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.662638  1234 tablet_bootstrap.cc:654] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.677356  1229 tablet_bootstrap.cc:654] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.682677   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 0dffb3eb778946869b914e5b05a67a69 (table default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]) successfully deleted
I20250628 07:52:35.688095  2206 tablet_replica.cc:331] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.689563  2206 raft_consensus.cc:2241] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.691490  2206 raft_consensus.cc:2270] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.699046  1230 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:35.699810  2206 ts_tablet_manager.cc:1905] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:35.707542   757 consensus_peers.cc:489] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:35.699435  1230 ts_tablet_manager.cc:1397] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.365s	user 0.001s	sys 0.008s
I20250628 07:52:35.711081   538 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_a6c8170745a346f9bc81b8e023e95b7c" } new_table_name: "default.Table_c47ff4f5324245c3b4b504c56d36649b" modify_external_catalogs: true
I20250628 07:52:35.715425  1230 raft_consensus.cc:357] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.716007  1230 raft_consensus.cc:383] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.716224  1230 raft_consensus.cc:738] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.720811  2180 consensus_queue.cc:1035] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.724300  1229 tablet_bootstrap.cc:492] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.724725  1229 ts_tablet_manager.cc:1397] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.085s	user 0.010s	sys 0.006s
I20250628 07:52:35.734665  1229 raft_consensus.cc:357] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.735303  1229 raft_consensus.cc:383] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.735530  1229 raft_consensus.cc:738] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.737558   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_6865e743213148aba2ee6d7058bc7219"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.739023   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6865e743213148aba2ee6d7058bc7219 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.736040  1229 consensus_queue.cc:260] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.743016  1229 ts_tablet_manager.cc:1428] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.018s	user 0.001s	sys 0.005s
I20250628 07:52:35.747418  1229 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.624919  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 319) took 1351 ms. Trace:
I20250628 07:52:35.754726  2237 rpcz_store.cc:276] 0628 07:52:34.273986 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.274493 (+   507us) service_pool.cc:225] Handling call
0628 07:52:35.225058 (+950565us) ts_tablet_manager.cc:1145] Deleting tablet 5475a89692eb450d82cde521ec8c9a73
0628 07:52:35.225116 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:35.374688 (+149572us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.623031 (+248343us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.624870 (+  1839us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":49689,"tablet-delete.queue_time_us":949894}
I20250628 07:52:35.756145   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 5475a89692eb450d82cde521ec8c9a73 (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]) successfully deleted
I20250628 07:52:35.728777  2257 ts_tablet_manager.cc:1939] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:35.758001   540 master_service.cc:948] ReplaceTablet: received request to replace tablet 8f7dde9b060f4d839724e5f718519fb1 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:35.759606  1229 tablet_bootstrap.cc:654] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.753767  2206 ts_tablet_manager.cc:1918] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:35.762461  2206 log.cc:1199] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:35.763204  2206 ts_tablet_manager.cc:1939] T cc0bf0edebb140209531ccf917fd1ed8 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.764026  2206 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 337) took 1052 ms. Trace:
I20250628 07:52:35.764369  2206 rpcz_store.cc:276] 0628 07:52:34.711259 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.711379 (+   120us) service_pool.cc:225] Handling call
0628 07:52:35.687906 (+976527us) ts_tablet_manager.cc:1145] Deleting tablet cc0bf0edebb140209531ccf917fd1ed8
0628 07:52:35.687960 (+    54us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:35.739270 (+ 51310us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.753572 (+ 14302us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.763975 (+ 10403us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":897209}
I20250628 07:52:35.765707   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet cc0bf0edebb140209531ccf917fd1ed8 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
I20250628 07:52:35.766489  2206 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 334) took 1084 ms. Trace:
I20250628 07:52:35.766788  2206 rpcz_store.cc:276] 0628 07:52:34.682404 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.683361 (+   957us) service_pool.cc:225] Handling call
0628 07:52:35.766285 (+1082924us) ts_tablet_manager.cc:1145] Deleting tablet 9579dd44e5e241aea2671be28e16ef7c
0628 07:52:35.766322 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:35.766448 (+   126us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":975068}
W20250628 07:52:35.767767   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because the tablet was not found. No further retry: Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:35.767287  2273 raft_consensus.cc:491] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:35.768234  2273 raft_consensus.cc:513] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.768620  2206 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 335) took 1086 ms. Trace:
I20250628 07:52:35.768976  2206 rpcz_store.cc:276] 0628 07:52:34.682684 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.789030 (+106346us) service_pool.cc:225] Handling call
0628 07:52:35.768412 (+979382us) ts_tablet_manager.cc:1145] Deleting tablet ef060774191146168ee8d1e13c9cd71b
0628 07:52:35.768446 (+    34us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:35.768569 (+   123us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":976651}
W20250628 07:52:35.770535   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ef060774191146168ee8d1e13c9cd71b because the tablet was not found. No further retry: Not found: Tablet not found: ef060774191146168ee8d1e13c9cd71b
I20250628 07:52:35.770673   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:35.771193   820 raft_consensus.cc:2466] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:35.771690  2206 tablet_replica.cc:331] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.772339   920 leader_election.cc:304] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:35.773423  1863 raft_consensus.cc:2802] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:35.773727  1863 raft_consensus.cc:491] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:35.774003  1863 raft_consensus.cc:3058] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.777479   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_7a076a51572f4693af02750b6627265e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.777838  2273 leader_election.cc:290] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.778743  2206 raft_consensus.cc:2241] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.778628  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:35.778831   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7a076a51572f4693af02750b6627265e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.779107  2206 raft_consensus.cc:2270] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.779163  1124 raft_consensus.cc:2466] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:35.781267  1229 tablet_bootstrap.cc:492] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.781626  1229 ts_tablet_manager.cc:1397] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.034s	user 0.011s	sys 0.000s
I20250628 07:52:35.783660  1229 raft_consensus.cc:357] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.784281  1229 raft_consensus.cc:383] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.784528  1229 raft_consensus.cc:738] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.786917   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_fd173dac22494e51a7adca1ec4817a45" } new_table_name: "default.Table_59e9b18b772e4d5dafd78313d4b47800" modify_external_catalogs: true
I20250628 07:52:35.786810  1229 consensus_queue.cc:260] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.789425  1229 ts_tablet_manager.cc:1428] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.004s	sys 0.002s
I20250628 07:52:35.790463  1229 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.794446  1928 consensus_queue.cc:1035] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.786242  2206 ts_tablet_manager.cc:1905] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:35.796604   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 102276us
I20250628 07:52:35.787024  1863 raft_consensus.cc:513] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.800406  1863 leader_election.cc:290] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:35.800467  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:35.800936  1125 raft_consensus.cc:3058] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.801064   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af13b56df157443c994f747e52b0f458" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:35.801535   820 raft_consensus.cc:3058] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.807101  1125 raft_consensus.cc:2466] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:35.807976   921 leader_election.cc:304] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:35.808411   820 raft_consensus.cc:2466] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:35.808709  2273 raft_consensus.cc:2802] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:35.809074  2273 raft_consensus.cc:695] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:35.810305  2273 consensus_queue.cc:237] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.817260  2237 tablet_replica.cc:331] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:35.818207  2237 raft_consensus.cc:2241] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:35.819159  2237 raft_consensus.cc:2270] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.822258  1229 tablet_bootstrap.cc:654] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.828894  2206 ts_tablet_manager.cc:1918] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:35.829406  2206 log.cc:1199] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:35.830111  2206 ts_tablet_manager.cc:1939] T fef70db0996f48c69574337f8f7ecf37 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:35.830998  1230 consensus_queue.cc:260] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.831445   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]) successfully deleted
W20250628 07:52:35.833006   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d3501090b0a34ca9882dde55b19e3621 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d3501090b0a34ca9882dde55b19e3621 already in progress: creating tablet
I20250628 07:52:35.833829  1230 ts_tablet_manager.cc:1428] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.125s	user 0.004s	sys 0.001s
W20250628 07:52:35.834821   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 8e3e0e52b4924e67904d3acfa4dd5752 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 8e3e0e52b4924e67904d3acfa4dd5752 already in progress: creating tablet
W20250628 07:52:35.836364   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 1c65fa809b17418c85d4d72ae1e0ee3a because the tablet was not found. No further retry: Not found: Tablet not found: 1c65fa809b17418c85d4d72ae1e0ee3a
W20250628 07:52:35.838071   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 5475a89692eb450d82cde521ec8c9a73 because the tablet was not found. No further retry: Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:35.839027  2206 tablet_replica.cc:331] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:35.840018  2206 raft_consensus.cc:2241] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.840621  2206 raft_consensus.cc:2270] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.850546   543 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_00d339485c8d4aa38db91013d26fdcc5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:35.852124   543 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_00d339485c8d4aa38db91013d26fdcc5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.853226  1234 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:35.853636  1234 ts_tablet_manager.cc:1397] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.265s	user 0.008s	sys 0.001s
I20250628 07:52:35.855623  2206 ts_tablet_manager.cc:1905] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.857530   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:35.858664   540 catalog_manager.cc:6652] ReplaceTablet: tablet 8f7dde9b060f4d839724e5f718519fb1 deleted and replaced by tablet aec37016d1674945b399cfc2f92b08fc
I20250628 07:52:35.859158   794 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 with delete_type TABLET_DATA_DELETED (replaced by tablet aec37016d1674945b399cfc2f92b08fc) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:35.860211  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 with delete_type TABLET_DATA_DELETED (replaced by tablet aec37016d1674945b399cfc2f92b08fc) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:35.861538   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 with delete_type TABLET_DATA_DELETED (replaced by tablet aec37016d1674945b399cfc2f92b08fc) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:35.883221  2238 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20250628 07:52:35.884197  2238 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Bootstrap complete.
I20250628 07:52:35.878590  2237 ts_tablet_manager.cc:1905] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.897837  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 310) took 1403 ms. Trace:
I20250628 07:52:35.898209  2257 rpcz_store.cc:276] 0628 07:52:34.494264 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.494370 (+   106us) service_pool.cc:225] Handling call
0628 07:52:34.961527 (+467157us) ts_tablet_manager.cc:1145] Deleting tablet 46ff641001cd4b01a4c09c0d8b79ebf3
0628 07:52:34.961576 (+    49us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:35.338881 (+377305us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.638196 (+299315us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:35.897707 (+259511us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":457360}
I20250628 07:52:35.899457  2257 tablet_replica.cc:331] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:35.899559   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (table default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]) successfully deleted
I20250628 07:52:35.900293  2257 raft_consensus.cc:2241] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:35.900808  2257 raft_consensus.cc:2270] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:35.905071  1230 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:35.910166   981 raft_consensus.cc:3058] T 08afbaee7cb74ce8a6b2958a6828eca1 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:35.922791  1229 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.922716   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_0b4dcd93c24b4479ad87780460b6afd8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:35.923190  1229 ts_tablet_manager.cc:1397] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.133s	user 0.004s	sys 0.006s
W20250628 07:52:35.924167   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0b4dcd93c24b4479ad87780460b6afd8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:35.925992  2025 consensus_queue.cc:1035] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:35.928233  1229 raft_consensus.cc:357] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.928894  1229 raft_consensus.cc:383] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.929155  1229 raft_consensus.cc:738] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.929769  1229 consensus_queue.cc:260] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.932158  1229 ts_tablet_manager.cc:1428] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.009s	user 0.002s	sys 0.004s
I20250628 07:52:35.933158  1229 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.943075  1229 tablet_bootstrap.cc:654] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.950026  2257 ts_tablet_manager.cc:1905] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:35.925173  1234 raft_consensus.cc:357] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.955662  1234 raft_consensus.cc:383] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.955993  1234 raft_consensus.cc:738] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.956732  1234 consensus_queue.cc:260] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.958987  1234 ts_tablet_manager.cc:1428] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.105s	user 0.005s	sys 0.000s
W20250628 07:52:35.968647   757 consensus_peers.cc:489] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:35.972397  1229 tablet_bootstrap.cc:492] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.972806  1229 ts_tablet_manager.cc:1397] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.040s	user 0.014s	sys 0.008s
I20250628 07:52:35.972986  1234 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:35.975113  1229 raft_consensus.cc:357] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.975757  1229 raft_consensus.cc:383] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:35.976029  1229 raft_consensus.cc:738] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:35.976682  1229 consensus_queue.cc:260] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:35.979092  1229 ts_tablet_manager.cc:1428] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20250628 07:52:35.980020  1229 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:35.986965  1229 tablet_bootstrap.cc:654] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:35.998289  1229 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:35.998699  1229 ts_tablet_manager.cc:1397] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.019s	user 0.014s	sys 0.000s
I20250628 07:52:36.001065  1229 raft_consensus.cc:357] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.003871  1229 raft_consensus.cc:383] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.004145  1229 raft_consensus.cc:738] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.004803  1229 consensus_queue.cc:260] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.006731  1229 ts_tablet_manager.cc:1428] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.008s	user 0.004s	sys 0.003s
I20250628 07:52:36.007591  1229 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:36.013151   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 50058.1us
I20250628 07:52:36.013520  1229 tablet_bootstrap.cc:654] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.021178  1229 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:36.021544  1229 ts_tablet_manager.cc:1397] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.014s	user 0.013s	sys 0.000s
I20250628 07:52:36.023640  1229 raft_consensus.cc:357] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.024214  1229 raft_consensus.cc:383] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.024448  1229 raft_consensus.cc:738] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.024951  1234 tablet_bootstrap.cc:654] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.025005  1229 consensus_queue.cc:260] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.026924  1229 ts_tablet_manager.cc:1428] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:36.031327  1229 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:36.038195   962 tablet_service.cc:1468] Processing CreateTablet for tablet 4a2409a1155441c89df8d04722cd35ed (DEFAULT_TABLE table=default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 [id=a01aff122c7047bc8a2b83587776adea]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:36.038023   543 catalog_manager.cc:5582] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:36.039142   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4a2409a1155441c89df8d04722cd35ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:36.039038   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:36.039577   794 tablet_service.cc:1468] Processing CreateTablet for tablet 4a2409a1155441c89df8d04722cd35ed (DEFAULT_TABLE table=default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 [id=a01aff122c7047bc8a2b83587776adea]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:36.039625   543 catalog_manager.cc:5582] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:36.040632   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4a2409a1155441c89df8d04722cd35ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:36.040977   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:36.041746   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:36.042281   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:36.044941  1229 tablet_bootstrap.cc:654] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.048250  1863 raft_consensus.cc:491] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.048628  1863 raft_consensus.cc:513] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.050194  1863 leader_election.cc:290] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.052681  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:36.053282  1124 raft_consensus.cc:2466] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:36.054950   921 leader_election.cc:304] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.057204   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:36.059427   920 leader_election.cc:343] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.061026   540 master_service.cc:948] ReplaceTablet: received request to replace tablet 5fa4299a9e0f4b409d04c8c5900901c0 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:36.070030   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 711843da7df74ba2bb22d386bc477a38
I20250628 07:52:36.072144  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 711843da7df74ba2bb22d386bc477a38 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:36.073366  1230 tablet_bootstrap.cc:654] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.073916   955 tablet_service.cc:1515] Processing DeleteTablet for tablet 711843da7df74ba2bb22d386bc477a38 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:36.074478   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 711843da7df74ba2bb22d386bc477a38 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:36.081589  1863 raft_consensus.cc:2802] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:36.081967  1863 raft_consensus.cc:491] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20250628 07:52:36.082182   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 79d15e600b8b460fab54eb21193f1f40 consensus state has no leader
I20250628 07:52:36.082329  1863 raft_consensus.cc:3058] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.084403  2162 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.084849  2162 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.086630  2162 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:36.088168   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.088745   981 raft_consensus.cc:2466] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:36.090560   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:36.091858  1062 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.092787  1062 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:36.093704  2162 raft_consensus.cc:2802] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:36.094009  2162 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:36.094292  2162 raft_consensus.cc:3058] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.098490  2257 ts_tablet_manager.cc:1918] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:36.098866  2257 log.cc:1199] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1c65fa809b17418c85d4d72ae1e0ee3a
I20250628 07:52:36.101636   544 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_848608eca1b140e5bb004562ebd3fde8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:36.104095  2257 ts_tablet_manager.cc:1939] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:36.104897  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 311) took 1449 ms. Trace:
I20250628 07:52:36.118753  1101 tablet_service.cc:1468] Processing CreateTablet for tablet 4a2409a1155441c89df8d04722cd35ed (DEFAULT_TABLE table=default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 [id=a01aff122c7047bc8a2b83587776adea]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:36.119791  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4a2409a1155441c89df8d04722cd35ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:36.105237  2257 rpcz_store.cc:276] 0628 07:52:34.655665 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.655866 (+   201us) service_pool.cc:225] Handling call
0628 07:52:35.899305 (+1243439us) ts_tablet_manager.cc:1145] Deleting tablet 1c65fa809b17418c85d4d72ae1e0ee3a
0628 07:52:35.899344 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.027611 (+128267us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.098273 (+ 70662us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.104843 (+  6570us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1242922}
I20250628 07:52:36.120738   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]) successfully deleted
I20250628 07:52:36.124640  2025 consensus_queue.cc:1035] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:36.125627  2257 tablet_replica.cc:331] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:36.149137   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 5fa4299a9e0f4b409d04c8c5900901c0
I20250628 07:52:36.158383   798 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:36.156616   544 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_848608eca1b140e5bb004562ebd3fde8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:36.165663   962 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:36.169502   540 catalog_manager.cc:6652] ReplaceTablet: tablet 5fa4299a9e0f4b409d04c8c5900901c0 deleted and replaced by tablet 0bdb4c499be441379663830f26d80204
I20250628 07:52:36.170686  1106 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:36.189004   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9ac8a7257de34accbdf87e0548648d25"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:36.190483   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9ac8a7257de34accbdf87e0548648d25 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:36.201436  2025 consensus_queue.cc:1035] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:36.201984  1033 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:36.223668  1863 raft_consensus.cc:513] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.226042   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:36.227411   920 leader_election.cc:343] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.250144  2162 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.251168  2257 raft_consensus.cc:2241] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.254101  2257 raft_consensus.cc:2270] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.254340  2206 ts_tablet_manager.cc:1918] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.254752  2206 log.cc:1199] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:36.255447  2206 ts_tablet_manager.cc:1939] T 83ffa65248fc4db3aecc5f91b648b83f P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:36.256557   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 83ffa65248fc4db3aecc5f91b648b83f (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:36.274847   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:36.276983  1062 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.283824  1230 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.284199  1230 ts_tablet_manager.cc:1397] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.379s	user 0.010s	sys 0.000s
W20250628 07:52:36.294445   757 consensus_peers.cc:489] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:36.321990  2206 tablet_replica.cc:331] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:36.322726  2206 raft_consensus.cc:2241] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.323163  2206 raft_consensus.cc:2270] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.330294  1863 leader_election.cc:290] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.329917  1230 raft_consensus.cc:357] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.330461  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:36.330673  1230 raft_consensus.cc:383] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.330991  1125 raft_consensus.cc:3058] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.330966  1230 raft_consensus.cc:738] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.331848  1230 consensus_queue.cc:260] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.332839   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:36.333352   981 raft_consensus.cc:3058] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.333755  2206 ts_tablet_manager.cc:1905] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.333884  1230 ts_tablet_manager.cc:1428] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.049s	user 0.007s	sys 0.000s
I20250628 07:52:36.334659  1230 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:36.336925  1125 raft_consensus.cc:2466] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:36.337581  2162 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:36.338250   921 leader_election.cc:304] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:36.339305  1863 raft_consensus.cc:2802] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:36.339615  2257 ts_tablet_manager.cc:1905] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.340008  1863 raft_consensus.cc:695] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:36.340695   981 raft_consensus.cc:2466] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:36.340718  1863 consensus_queue.cc:237] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.345513   982 raft_consensus.cc:3058] T aea9552ebd5949c8b0b5f23e132e9b6c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.366380  1229 tablet_bootstrap.cc:492] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:36.367436  1229 ts_tablet_manager.cc:1397] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.339s	user 0.114s	sys 0.148s
I20250628 07:52:36.367974  2257 ts_tablet_manager.cc:1918] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:36.368350  2257 log.cc:1199] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:36.377624  2257 ts_tablet_manager.cc:1939] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:36.378404  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 314) took 1678 ms. Trace:
I20250628 07:52:36.379710  1229 raft_consensus.cc:357] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.383543  1229 raft_consensus.cc:383] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.387146  1229 raft_consensus.cc:738] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.378749  2257 rpcz_store.cc:276] 0628 07:52:34.700152 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.700253 (+   101us) service_pool.cc:225] Handling call
0628 07:52:36.125405 (+1425152us) ts_tablet_manager.cc:1145] Deleting tablet 2ae890710f7f44cf899dd19e96fa9181
0628 07:52:36.125469 (+    64us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.253764 (+128295us) spinlock_profiling.cc:243] Waited 124 ms on lock 0x7b54000e3dd4. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fbaa5fc15d1 00007fbaa6045092 00007fbaa646462c 00007fbaa7077d1a 00007fbaa70a7b70 00007fbaa708ef4d 00007fbaa708eef9 00007fbaa708ee89 00007fbaa708ee51 00007fbaa708e07d 000000000059bcd4
0628 07:52:36.348194 (+ 94430us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.367756 (+ 19562us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.378350 (+ 10594us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":286304128,"tablet-delete.queue_time_us":1424585}
I20250628 07:52:36.395380   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 2ae890710f7f44cf899dd19e96fa9181 (table default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]) successfully deleted
I20250628 07:52:36.396299  2257 tablet_replica.cc:331] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:36.390439  1229 consensus_queue.cc:260] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.398125   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c22cd1bf6d884c33bf39513fb5da717a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:36.399562   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c22cd1bf6d884c33bf39513fb5da717a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:36.400210  2257 raft_consensus.cc:2241] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:36.402877  2257 raft_consensus.cc:2270] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.403949   544 master_service.cc:948] ReplaceTablet: received request to replace tablet af28ae2139b2417381808cb4068ddf60 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:36.410261  1062 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
W20250628 07:52:36.410907  1062 raft_consensus.cc:2684] T 2ae890710f7f44cf899dd19e96fa9181 P 6a58514e01ea4989809bc2992fb2b806: unable to run election callback: Service unavailable: Thread pool token was shut down
I20250628 07:52:36.412992  1229 ts_tablet_manager.cc:1428] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.045s	user 0.008s	sys 0.000s
I20250628 07:52:36.416921   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:36.417140  1229 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:36.417747   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:36.418594   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:36.419183   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]). Expected version 3 got 0
I20250628 07:52:36.402104  1230 tablet_bootstrap.cc:654] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.433847  2257 ts_tablet_manager.cc:1905] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.441156  1229 tablet_bootstrap.cc:654] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.421411  2238 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:36.452142  2147 raft_consensus.cc:491] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 82f43cdfab544b17a6a71033983e65d4)
I20250628 07:52:36.452606  2147 raft_consensus.cc:513] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.451196  2206 ts_tablet_manager.cc:1918] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.454332  2147 leader_election.cc:290] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.450718  2238 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5fd67221fa5f424ea0f83e582629d276, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.456501  2206 log.cc:1199] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/b51b51eb951d45a78bd2bffe47834b95
W20250628 07:52:36.458112   758 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:36.462253  1230 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.462867  1230 ts_tablet_manager.cc:1397] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.128s	user 0.007s	sys 0.002s
W20250628 07:52:36.463944   757 leader_election.cc:343] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:36.465301  1230 raft_consensus.cc:357] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.465004  2206 ts_tablet_manager.cc:1939] T b51b51eb951d45a78bd2bffe47834b95 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:36.467608  1230 raft_consensus.cc:383] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.467865  1230 raft_consensus.cc:738] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.463493  2238 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 3.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:36.464284   757 leader_election.cc:304] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:36.427433  2180 consensus_queue.cc:1035] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.035s
I20250628 07:52:36.472455  1230 consensus_queue.cc:260] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.476552  2025 raft_consensus.cc:2747] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:36.490562  1230 ts_tablet_manager.cc:1428] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.027s	user 0.004s	sys 0.000s
I20250628 07:52:36.491472  1230 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:36.491602   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet b51b51eb951d45a78bd2bffe47834b95 (table default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]) successfully deleted
I20250628 07:52:36.492383  2206 tablet_replica.cc:331] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:36.493074  2206 raft_consensus.cc:2241] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:36.493846  2206 raft_consensus.cc:2270] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.500339   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet af28ae2139b2417381808cb4068ddf60
I20250628 07:52:36.501559   544 catalog_manager.cc:6652] ReplaceTablet: tablet af28ae2139b2417381808cb4068ddf60 deleted and replaced by tablet 7e3210014d264071bfc2960a7406bcd7
I20250628 07:52:36.502921  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet af28ae2139b2417381808cb4068ddf60 with delete_type TABLET_DATA_DELETED (replaced by tablet 7e3210014d264071bfc2960a7406bcd7) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:36.503736   962 tablet_service.cc:1515] Processing DeleteTablet for tablet af28ae2139b2417381808cb4068ddf60 with delete_type TABLET_DATA_DELETED (replaced by tablet 7e3210014d264071bfc2960a7406bcd7) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:36.499181  2328 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:36.507829  2328 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:36.509582   546 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:36.511389  2206 ts_tablet_manager.cc:1905] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.515311  2238 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:52:36.521106   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9f45c9f9b24e4c9782068f570ce3f3e7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:36.534127  1230 tablet_bootstrap.cc:654] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.534473  1234 tablet_bootstrap.cc:492] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:36.534921  1234 ts_tablet_manager.cc:1397] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.562s	user 0.009s	sys 0.006s
I20250628 07:52:36.538380  2257 ts_tablet_manager.cc:1918] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.538828  2257 log.cc:1199] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:36.539602  2257 ts_tablet_manager.cc:1939] T ee03996105074308bda56d36ba3a1c5d P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:36.550482   798 tablet_service.cc:1515] Processing DeleteTablet for tablet af28ae2139b2417381808cb4068ddf60 with delete_type TABLET_DATA_DELETED (replaced by tablet 7e3210014d264071bfc2960a7406bcd7) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:36.552013  2237 ts_tablet_manager.cc:1918] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.552230  2175 raft_consensus.cc:491] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.552512  2237 log.cc:1199] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:36.552672  2175 raft_consensus.cc:513] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.553475  2237 ts_tablet_manager.cc:1939] T 0dffb3eb778946869b914e5b05a67a69 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:36.554661  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 321) took 2275 ms. Trace:
I20250628 07:52:36.555032  2237 rpcz_store.cc:276] 0628 07:52:34.278843 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.278939 (+    96us) service_pool.cc:225] Handling call
0628 07:52:35.756807 (+1477868us) ts_tablet_manager.cc:1145] Deleting tablet 0dffb3eb778946869b914e5b05a67a69
0628 07:52:35.817144 (+ 60337us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.192882 (+375738us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.535423 (+342541us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.554598 (+ 19175us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1477325}
I20250628 07:52:36.556257   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.556499  2237 tablet_replica.cc:331] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:36.556795  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 313) took 1856 ms. Trace:
I20250628 07:52:36.556843   982 raft_consensus.cc:2466] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:36.558617   757 leader_election.cc:304] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.559326  2330 raft_consensus.cc:2802] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:36.559628  2330 raft_consensus.cc:491] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:36.559918  2330 raft_consensus.cc:3058] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.565006  1863 raft_consensus.cc:491] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.565292  2175 leader_election.cc:290] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.565478  1863 raft_consensus.cc:513] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.566063  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:36.566625  1124 raft_consensus.cc:2466] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:36.557077  2257 rpcz_store.cc:276] 0628 07:52:34.699883 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.700333 (+   450us) service_pool.cc:225] Handling call
0628 07:52:36.396146 (+1695813us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:36.396188 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.447546 (+ 51358us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.538114 (+ 90568us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.556717 (+ 18603us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1695256}
I20250628 07:52:36.567924   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet ee03996105074308bda56d36ba3a1c5d (table default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]) successfully deleted
I20250628 07:52:36.568087  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:36.568594  1125 raft_consensus.cc:2466] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:36.569649  1863 leader_election.cc:290] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.570209   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 0dffb3eb778946869b914e5b05a67a69 (table default.Table_09cc8edc019c4d26936ec96d72adbb0d [id=4f2d26d9e9bc44a2a2620ec0ec96f3c1]) successfully deleted
I20250628 07:52:36.570747   921 leader_election.cc:304] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.571643  1863 raft_consensus.cc:2802] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:36.571256  2330 raft_consensus.cc:513] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.571959  1863 raft_consensus.cc:491] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:36.572355  1863 raft_consensus.cc:3058] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.573782   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:36.574328   982 raft_consensus.cc:3058] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.577592  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 312) took 1878 ms. Trace:
I20250628 07:52:36.577914  2257 rpcz_store.cc:276] 0628 07:52:34.699574 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.705180 (+  5606us) service_pool.cc:225] Handling call
0628 07:52:36.577353 (+1872173us) ts_tablet_manager.cc:1145] Deleting tablet 0dffb3eb778946869b914e5b05a67a69
0628 07:52:36.577392 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.577543 (+   151us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1871572}
I20250628 07:52:36.578362  1234 raft_consensus.cc:357] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:36.578984   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because the tablet was not found. No further retry: Not found: Tablet not found: 0dffb3eb778946869b914e5b05a67a69
I20250628 07:52:36.579123  1234 raft_consensus.cc:383] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.579366  1234 raft_consensus.cc:738] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.579775  2162 raft_consensus.cc:491] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.580183  2162 raft_consensus.cc:513] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.580093  1234 consensus_queue.cc:260] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.581933  2162 leader_election.cc:290] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:36.583438   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.584378  2257 tablet_replica.cc:331] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:36.585135  2257 raft_consensus.cc:2241] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:36.550123   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9f45c9f9b24e4c9782068f570ce3f3e7 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:36.585336  1062 leader_election.cc:343] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.585562  2257 raft_consensus.cc:2270] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.586171  1234 ts_tablet_manager.cc:1428] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.051s	user 0.004s	sys 0.000s
I20250628 07:52:36.587023  1234 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:36.587339  2257 ts_tablet_manager.cc:1905] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.591588  1863 raft_consensus.cc:513] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.593441  2237 raft_consensus.cc:2241] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.593990  2237 raft_consensus.cc:2270] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.598287  2273 raft_consensus.cc:491] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.598703  2273 raft_consensus.cc:513] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.600523  2273 leader_election.cc:290] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.601976  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:36.602511  1125 raft_consensus.cc:2466] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:36.603829   921 leader_election.cc:304] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.605175  2273 raft_consensus.cc:2802] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:36.605473  2273 raft_consensus.cc:491] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:36.605759  2273 raft_consensus.cc:3058] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.611989   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:36.612536   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:36.618017  2237 ts_tablet_manager.cc:1905] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.619777   982 raft_consensus.cc:2466] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:36.622464   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:36.622884   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:36.625105  1062 leader_election.cc:343] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.625464  1062 leader_election.cc:304] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:36.626820  2162 raft_consensus.cc:2747] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:36.627707   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:36.639425   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:36.640185   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:36.640743   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:36.641800   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:36.642287  1230 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.643154  1230 ts_tablet_manager.cc:1397] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.152s	user 0.018s	sys 0.005s
I20250628 07:52:36.643445   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:36.644062   541 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]). Expected version 1 got 0
I20250628 07:52:36.645076  1230 raft_consensus.cc:357] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.645663  1230 raft_consensus.cc:383] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.645884  1230 raft_consensus.cc:738] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.646422  1230 consensus_queue.cc:260] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.647818  1230 ts_tablet_manager.cc:1428] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.004s	user 0.002s	sys 0.000s
I20250628 07:52:36.644145  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d87dd96d00964e12acd6e380a62c94d8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:36.648424  1124 raft_consensus.cc:3058] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.645284  2330 leader_election.cc:290] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.647073   757 leader_election.cc:304] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.649464  2175 raft_consensus.cc:2802] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:36.649966  2175 raft_consensus.cc:695] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:36.650823  2175 consensus_queue.cc:237] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.632136  1863 leader_election.cc:290] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.664791  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:36.665938  1863 consensus_queue.cc:1035] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:36.666553  1125 raft_consensus.cc:3058] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.682801  2257 ts_tablet_manager.cc:1918] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.683168  2257 log.cc:1199] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:36.683844  2257 ts_tablet_manager.cc:1939] T fef70db0996f48c69574337f8f7ecf37 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:36.685817  2330 consensus_queue.cc:1035] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:36.686940  1229 tablet_bootstrap.cc:492] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:36.687391  1229 ts_tablet_manager.cc:1397] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.271s	user 0.025s	sys 0.028s
I20250628 07:52:36.648622  1230 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:36.687963   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 58336us
I20250628 07:52:36.689898   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:36.690074  1229 raft_consensus.cc:357] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.690691  1229 raft_consensus.cc:383] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.690937  1229 raft_consensus.cc:738] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.691471  1229 consensus_queue.cc:260] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:36.697988   920 leader_election.cc:343] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:36.699005   920 leader_election.cc:343] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.702080  1230 tablet_bootstrap.cc:654] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:36.699782   920 leader_election.cc:343] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.706583  1229 ts_tablet_manager.cc:1428] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.019s	user 0.006s	sys 0.000s
I20250628 07:52:36.706403  2273 raft_consensus.cc:513] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.706920  2338 consensus_queue.cc:1035] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:36.707527  1229 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:36.708618  2273 leader_election.cc:290] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.709038  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 315) took 1837 ms. Trace:
I20250628 07:52:36.709393  2257 rpcz_store.cc:276] 0628 07:52:34.871703 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.871794 (+    91us) service_pool.cc:225] Handling call
0628 07:52:36.584245 (+1712451us) ts_tablet_manager.cc:1145] Deleting tablet fef70db0996f48c69574337f8f7ecf37
0628 07:52:36.584280 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.638323 (+ 54043us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.682563 (+ 44240us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.708980 (+ 26417us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1711877}
I20250628 07:52:36.709920  1234 tablet_bootstrap.cc:654] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.710239  1123 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:36.710834  1123 raft_consensus.cc:3058] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.711858   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]) successfully deleted
I20250628 07:52:36.712785  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 316) took 1751 ms. Trace:
I20250628 07:52:36.713068  2257 rpcz_store.cc:276] 0628 07:52:34.961020 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.961179 (+   159us) service_pool.cc:225] Handling call
0628 07:52:36.712496 (+1751317us) ts_tablet_manager.cc:1145] Deleting tablet d3501090b0a34ca9882dde55b19e3621
0628 07:52:36.712533 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.712738 (+   205us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1750724}
W20250628 07:52:36.714087   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d3501090b0a34ca9882dde55b19e3621 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d3501090b0a34ca9882dde55b19e3621 already in progress: creating tablet
I20250628 07:52:36.717429  2257 tablet_replica.cc:331] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:36.732095  1123 raft_consensus.cc:2466] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:36.718194  2257 raft_consensus.cc:2241] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.733754  2257 raft_consensus.cc:2270] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.733167   921 leader_election.cc:304] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:36.738420  2337 raft_consensus.cc:2802] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:36.738799  2257 ts_tablet_manager.cc:1905] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.738951  2337 raft_consensus.cc:695] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:36.739807  2337 consensus_queue.cc:237] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.743101  1125 raft_consensus.cc:2466] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:36.744040   921 leader_election.cc:304] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:36.745277  2147 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.745712  2147 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.745184  2335 raft_consensus.cc:2802] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:36.746573  2335 raft_consensus.cc:695] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:36.747427  2147 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:36.747323  2335 consensus_queue.cc:237] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.751386   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.752089   981 raft_consensus.cc:2391] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:36.755559  2330 consensus_queue.cc:1035] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:36.762233   758 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:36.762609   758 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:36.763456  2330 raft_consensus.cc:3058] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:36.774015  2330 raft_consensus.cc:2747] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:36.776259   542 catalog_manager.cc:5582] T 08afbaee7cb74ce8a6b2958a6828eca1 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:36.786692   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:36.787436   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:36.788218   542 catalog_manager.cc:5582] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:36.790491  1230 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.790917  1230 ts_tablet_manager.cc:1397] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.142s	user 0.009s	sys 0.000s
I20250628 07:52:36.790434   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:36.792279   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:36.793004   542 catalog_manager.cc:5582] T 39ba8057357241b99882e883f6af93e7 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:36.793689  1230 raft_consensus.cc:357] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.794423  1230 raft_consensus.cc:383] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.794662  1230 raft_consensus.cc:738] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.795364   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:36.796983   542 catalog_manager.cc:5582] T aea9552ebd5949c8b0b5f23e132e9b6c P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:36.797230  1230 consensus_queue.cc:260] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.798204  1229 tablet_bootstrap.cc:654] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.798326   542 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:36.806178  1230 ts_tablet_manager.cc:1428] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.015s	user 0.008s	sys 0.000s
I20250628 07:52:36.806986  1230 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:36.818161   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fa3ec14241e344029df3e39473181078" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:36.820017   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_af13e11b58c24735a4b8dbb9a7fefa22" } modify_external_catalogs: true
I20250628 07:52:36.820549   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_af13e11b58c24735a4b8dbb9a7fefa22" } modify_external_catalogs: true
I20250628 07:52:36.832562  1230 tablet_bootstrap.cc:654] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:36.844229   920 leader_election.cc:343] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.846820  2257 ts_tablet_manager.cc:1918] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:36.850977  1234 tablet_bootstrap.cc:492] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:36.851383  1234 ts_tablet_manager.cc:1397] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.265s	user 0.009s	sys 0.000s
I20250628 07:52:36.851361  2257 log.cc:1199] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:36.852588  2257 ts_tablet_manager.cc:1939] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:36.854521  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 317) took 1648 ms. Trace:
I20250628 07:52:36.854880  2257 rpcz_store.cc:276] 0628 07:52:35.206321 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.206436 (+   115us) service_pool.cc:225] Handling call
0628 07:52:36.717253 (+1510817us) ts_tablet_manager.cc:1145] Deleting tablet 8e3e0e52b4924e67904d3acfa4dd5752
0628 07:52:36.717296 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.813841 (+ 96545us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.844508 (+ 30667us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.854463 (+  9955us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1510298}
I20250628 07:52:36.856222   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 8e3e0e52b4924e67904d3acfa4dd5752 (table default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]) successfully deleted
I20250628 07:52:36.857311  2257 tablet_replica.cc:331] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:36.858556  2257 raft_consensus.cc:2241] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.861598  2257 raft_consensus.cc:2270] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:36.863889  2257 ts_tablet_manager.cc:1905] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:36.867647   757 consensus_peers.cc:489] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b51b51eb951d45a78bd2bffe47834b95. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:36.839363  1124 raft_consensus.cc:2466] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:36.885907  1234 raft_consensus.cc:357] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.886588  1234 raft_consensus.cc:383] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.886848  1234 raft_consensus.cc:738] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.899910   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet f0c00063b47e49099a2c0ab9c8587c04
I20250628 07:52:36.902653   794 tablet_service.cc:1515] Processing DeleteTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:36 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:36.903913   962 tablet_service.cc:1515] Processing DeleteTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:36 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:36.913724  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet f0c00063b47e49099a2c0ab9c8587c04 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:36 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:36.915149  1230 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.915598  1230 ts_tablet_manager.cc:1397] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.109s	user 0.010s	sys 0.002s
I20250628 07:52:36.917876  1230 raft_consensus.cc:357] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.918459  1230 raft_consensus.cc:383] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.918702  1230 raft_consensus.cc:738] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.919167  2206 ts_tablet_manager.cc:1918] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:36.919523  2206 log.cc:1199] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:36.919243  1230 consensus_queue.cc:260] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.920200  2206 ts_tablet_manager.cc:1939] T 8f7dde9b060f4d839724e5f718519fb1 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:36.924134  2237 ts_tablet_manager.cc:1918] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:36.938561  1230 ts_tablet_manager.cc:1428] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.023s	user 0.006s	sys 0.000s
I20250628 07:52:36.939436  1230 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:36.920929  2206 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 345) took 1059 ms. Trace:
I20250628 07:52:36.953747  2206 rpcz_store.cc:276] 0628 07:52:35.861340 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.861430 (+    90us) service_pool.cc:225] Handling call
0628 07:52:36.492253 (+630823us) ts_tablet_manager.cc:1145] Deleting tablet 8f7dde9b060f4d839724e5f718519fb1
0628 07:52:36.492289 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.668621 (+176332us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.918932 (+250311us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.920881 (+  1949us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":630344}
I20250628 07:52:36.955087   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
I20250628 07:52:36.948908  1230 tablet_bootstrap.cc:654] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:36.956005   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 711843da7df74ba2bb22d386bc477a38 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 711843da7df74ba2bb22d386bc477a38 already in progress: creating tablet
I20250628 07:52:36.961488  2206 tablet_replica.cc:331] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:36.962272  2206 raft_consensus.cc:2241] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:36.964900  2162 raft_consensus.cc:491] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.965333  2162 raft_consensus.cc:513] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.967309  2162 leader_election.cc:290] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:36.968406  1230 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:36.968791  1230 ts_tablet_manager.cc:1397] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.030s	user 0.008s	sys 0.004s
I20250628 07:52:36.971904  1230 raft_consensus.cc:357] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.972505  1230 raft_consensus.cc:383] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:36.972762  1230 raft_consensus.cc:738] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:36.973018  2162 raft_consensus.cc:491] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:36.973313  1230 consensus_queue.cc:260] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.973426  2162 raft_consensus.cc:513] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.975145  2162 leader_election.cc:290] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:36.977273  2237 log.cc:1199] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:36.977304   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:36.978145  2237 ts_tablet_manager.cc:1939] T 9579dd44e5e241aea2671be28e16ef7c P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:36.933343  1234 consensus_queue.cc:260] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:36.968894   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:36.980512  1234 ts_tablet_manager.cc:1428] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.129s	user 0.006s	sys 0.001s
W20250628 07:52:36.974205   758 consensus_peers.cc:489] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:36.980787  1062 leader_election.cc:343] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:36.981398  1062 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.986557  1230 ts_tablet_manager.cc:1428] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.018s	user 0.003s	sys 0.004s
I20250628 07:52:36.994134  1230 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:36.970077   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.978382   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:36.995622  2206 raft_consensus.cc:2270] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:36.996855  1062 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.997246  1062 leader_election.cc:304] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:36.997460  2206 ts_tablet_manager.cc:1905] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:36.997848  2364 raft_consensus.cc:2747] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:36.998135  1062 leader_election.cc:343] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:36.998471  1062 leader_election.cc:304] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:36.999032  2364 raft_consensus.cc:2747] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:37.001729  1230 tablet_bootstrap.cc:654] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:36.984236  1229 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:37.006582  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 320) took 2728 ms. Trace:
I20250628 07:52:37.006989  2237 rpcz_store.cc:276] 0628 07:52:34.278549 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.280651 (+  2102us) service_pool.cc:225] Handling call
0628 07:52:36.556359 (+2275708us) ts_tablet_manager.cc:1145] Deleting tablet 9579dd44e5e241aea2671be28e16ef7c
0628 07:52:36.556398 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.781408 (+225010us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:36.923907 (+142499us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.006523 (+ 82616us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":35989,"tablet-delete.queue_time_us":2275126}
I20250628 07:52:37.008378   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 9579dd44e5e241aea2671be28e16ef7c (table default.Table_6d0f8ad1bc934dbeb1ec11510bc277a3 [id=95c8ce70349043df924705decc86478c]) successfully deleted
I20250628 07:52:37.009490  1234 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:37.017413  1229 ts_tablet_manager.cc:1397] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.310s	user 0.010s	sys 0.002s
I20250628 07:52:37.017445  2237 tablet_replica.cc:331] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:37.018967  2237 raft_consensus.cc:2241] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:37.019889  2237 raft_consensus.cc:2270] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.020447  1229 raft_consensus.cc:357] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.022965  1234 tablet_bootstrap.cc:654] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.023267  2257 ts_tablet_manager.cc:1918] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:37.023656  2257 log.cc:1199] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:37.024365  2257 ts_tablet_manager.cc:1939] T 8f7dde9b060f4d839724e5f718519fb1 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
W20250628 07:52:37.047425   757 consensus_peers.cc:489] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:37.052383  2237 ts_tablet_manager.cc:1905] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.065380  1229 raft_consensus.cc:383] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.065825  1229 raft_consensus.cc:738] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.066526  1229 consensus_queue.cc:260] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.069013  1229 ts_tablet_manager.cc:1428] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.051s	user 0.003s	sys 0.001s
I20250628 07:52:37.075146  1229 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:37.080313  1229 tablet_bootstrap.cc:654] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.083333  1234 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:37.083765  1234 ts_tablet_manager.cc:1397] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.074s	user 0.008s	sys 0.000s
I20250628 07:52:37.089974  1234 raft_consensus.cc:357] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.090708  1234 raft_consensus.cc:383] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.090976  1234 raft_consensus.cc:738] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.091861  2206 ts_tablet_manager.cc:1918] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.092234  2206 log.cc:1199] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/5fa4299a9e0f4b409d04c8c5900901c0
I20250628 07:52:37.092907  2206 ts_tablet_manager.cc:1939] T 5fa4299a9e0f4b409d04c8c5900901c0 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:37.104209   794 tablet_service.cc:1468] Processing CreateTablet for tablet c4e22ae1341d4262ad8e446315fae0f8 (DEFAULT_TABLE table=default.Table_7a076a51572f4693af02750b6627265e [id=7b1235959a5c4e3b9d4c22edcbba14d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.105361   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4e22ae1341d4262ad8e446315fae0f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.096681   962 tablet_service.cc:1468] Processing CreateTablet for tablet c4e22ae1341d4262ad8e446315fae0f8 (DEFAULT_TABLE table=default.Table_7a076a51572f4693af02750b6627265e [id=7b1235959a5c4e3b9d4c22edcbba14d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.105363  1234 consensus_queue.cc:260] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.110033   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4e22ae1341d4262ad8e446315fae0f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.116037  1234 ts_tablet_manager.cc:1428] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.000s
I20250628 07:52:37.116881  1234 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:37.122463   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:37.123162   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:37.123838   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:37.124419   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:37.137327  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 318) took 1277 ms. Trace:
I20250628 07:52:37.137704  2257 rpcz_store.cc:276] 0628 07:52:35.859968 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.860094 (+   126us) service_pool.cc:225] Handling call
0628 07:52:36.857176 (+997082us) ts_tablet_manager.cc:1145] Deleting tablet 8f7dde9b060f4d839724e5f718519fb1
0628 07:52:36.857214 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:36.955943 (+ 98729us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.022992 (+ 67049us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.137263 (+114271us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":996568}
I20250628 07:52:37.139154   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
I20250628 07:52:37.140532  1234 tablet_bootstrap.cc:654] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.146155   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 5fa4299a9e0f4b409d04c8c5900901c0 (table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]) successfully deleted
W20250628 07:52:37.148219   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet af28ae2139b2417381808cb4068ddf60 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet af28ae2139b2417381808cb4068ddf60 already in progress: creating tablet
W20250628 07:52:37.149950   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet f0c00063b47e49099a2c0ab9c8587c04 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f0c00063b47e49099a2c0ab9c8587c04 already in progress: creating tablet
I20250628 07:52:37.153301  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 319) took 1081 ms. Trace:
I20250628 07:52:37.153635  2257 rpcz_store.cc:276] 0628 07:52:36.071895 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.072020 (+   125us) service_pool.cc:225] Handling call
0628 07:52:37.139845 (+1067825us) ts_tablet_manager.cc:1145] Deleting tablet 711843da7df74ba2bb22d386bc477a38
0628 07:52:37.139886 (+    41us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.153243 (+ 13357us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1067327}
W20250628 07:52:37.154707   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 711843da7df74ba2bb22d386bc477a38 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 711843da7df74ba2bb22d386bc477a38 already in progress: creating tablet
I20250628 07:52:37.155489  2257 tablet_replica.cc:331] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:37.156199  2257 raft_consensus.cc:2241] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:37.156600  2257 raft_consensus.cc:2270] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.158294  2257 ts_tablet_manager.cc:1905] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.162688  1230 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:37.163061  1230 ts_tablet_manager.cc:1397] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.169s	user 0.019s	sys 0.007s
I20250628 07:52:37.166949  1101 tablet_service.cc:1468] Processing CreateTablet for tablet c4e22ae1341d4262ad8e446315fae0f8 (DEFAULT_TABLE table=default.Table_7a076a51572f4693af02750b6627265e [id=7b1235959a5c4e3b9d4c22edcbba14d6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.167919  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c4e22ae1341d4262ad8e446315fae0f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.173333  2175 raft_consensus.cc:491] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.173740  2175 raft_consensus.cc:513] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.175422  2237 ts_tablet_manager.cc:1918] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.175777  2237 log.cc:1199] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/b51b51eb951d45a78bd2bffe47834b95
I20250628 07:52:37.176083   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:37.176462  2237 ts_tablet_manager.cc:1939] T b51b51eb951d45a78bd2bffe47834b95 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:37.178602  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "95fa7c0a68d34d728042fb5a8a3b7307" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:37.179364  1124 raft_consensus.cc:2391] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:37.180745   758 leader_election.cc:304] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:37.181486  2330 raft_consensus.cc:3058] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:37.181955  2175 leader_election.cc:290] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:37.165050  1230 raft_consensus.cc:357] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.189796  1230 raft_consensus.cc:383] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.190093  1230 raft_consensus.cc:738] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.191234  2364 raft_consensus.cc:491] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.191660  2364 raft_consensus.cc:513] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.192523  2330 raft_consensus.cc:2747] T 95fa7c0a68d34d728042fb5a8a3b7307 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:37.193581  2364 leader_election.cc:290] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:37.190759  1230 consensus_queue.cc:260] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.194527   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8155990eebd84fcf9ebc357e186d1bf2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:37.196063  1230 ts_tablet_manager.cc:1428] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.033s	user 0.005s	sys 0.000s
I20250628 07:52:37.198539   798 tablet_service.cc:1468] Processing CreateTablet for tablet 6fdda1e4f7af468982a813f1ac023e01 (DEFAULT_TABLE table=default.Table_6865e743213148aba2ee6d7058bc7219 [id=c9e6ae6d71a04ae99e69992a0b198c52]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.199275   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8155990eebd84fcf9ebc357e186d1bf2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:37.202023   793 tablet_service.cc:1468] Processing CreateTablet for tablet f91208e5de0b42a5a3e17f26f423ea03 (DEFAULT_TABLE table=default.Table_8dcb758f32dc4345acc05a8e494ea755 [id=cc7adfe6bb724dc79d3fa8c5e4a24ded]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.203744   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6fdda1e4f7af468982a813f1ac023e01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.203969   955 tablet_service.cc:1468] Processing CreateTablet for tablet f91208e5de0b42a5a3e17f26f423ea03 (DEFAULT_TABLE table=default.Table_8dcb758f32dc4345acc05a8e494ea755 [id=cc7adfe6bb724dc79d3fa8c5e4a24ded]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.204931   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f91208e5de0b42a5a3e17f26f423ea03. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.205138   793 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f91208e5de0b42a5a3e17f26f423ea03. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.211583  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 324) took 2822 ms. Trace:
I20250628 07:52:37.211977  2237 rpcz_store.cc:276] 0628 07:52:34.389319 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.394533 (+  5214us) service_pool.cc:225] Handling call
0628 07:52:37.017282 (+2622749us) ts_tablet_manager.cc:1145] Deleting tablet b51b51eb951d45a78bd2bffe47834b95
0628 07:52:37.017327 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.119011 (+101684us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.175198 (+ 56187us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.211520 (+ 36322us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2622106}
I20250628 07:52:37.213129   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet b51b51eb951d45a78bd2bffe47834b95 (table default.Table_915f3db7b34449c4b8124af06f772bc8 [id=a01cdbc8466a4cbdb0c8279aceac76d7]) successfully deleted
I20250628 07:52:37.220551   982 raft_consensus.cc:3058] T 39ba8057357241b99882e883f6af93e7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:37.222157   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:37.223268   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:37.223968   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:37.224387   544 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]). Expected version 1 got 0
I20250628 07:52:37.230062  2237 tablet_replica.cc:331] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
W20250628 07:52:37.231335   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet d9e0ea5e300347c7adc8e30b9070976b consensus state has no leader
I20250628 07:52:37.231797   962 tablet_service.cc:1468] Processing CreateTablet for tablet 6fdda1e4f7af468982a813f1ac023e01 (DEFAULT_TABLE table=default.Table_6865e743213148aba2ee6d7058bc7219 [id=c9e6ae6d71a04ae99e69992a0b198c52]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.232890   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6fdda1e4f7af468982a813f1ac023e01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.200587  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 6fdda1e4f7af468982a813f1ac023e01 (DEFAULT_TABLE table=default.Table_6865e743213148aba2ee6d7058bc7219 [id=c9e6ae6d71a04ae99e69992a0b198c52]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.204298  1100 tablet_service.cc:1468] Processing CreateTablet for tablet f91208e5de0b42a5a3e17f26f423ea03 (DEFAULT_TABLE table=default.Table_8dcb758f32dc4345acc05a8e494ea755 [id=cc7adfe6bb724dc79d3fa8c5e4a24ded]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.233891  1230 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:37.209499  1062 leader_election.cc:343] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:37.234277  2237 raft_consensus.cc:2241] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:37.234879  1062 leader_election.cc:343] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:37.235203  1062 leader_election.cc:304] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:37.235814  2364 raft_consensus.cc:2747] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:37.237723   795 tablet_service.cc:1468] Processing CreateTablet for tablet 35fcce92db9a4e4a81f9d9db9d6283b8 (DEFAULT_TABLE table=default.Table_1cf78a5cb3c5490bae2317600c643635 [id=2613a92dda0d4236b4a4422cf612a6c6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.237880  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 35fcce92db9a4e4a81f9d9db9d6283b8 (DEFAULT_TABLE table=default.Table_1cf78a5cb3c5490bae2317600c643635 [id=2613a92dda0d4236b4a4422cf612a6c6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.238687   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fcce92db9a4e4a81f9d9db9d6283b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.233903  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6fdda1e4f7af468982a813f1ac023e01. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.243963  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fcce92db9a4e4a81f9d9db9d6283b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.244432  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f91208e5de0b42a5a3e17f26f423ea03. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.234759  2237 raft_consensus.cc:2270] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.255424  2237 ts_tablet_manager.cc:1905] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.259033  1230 tablet_bootstrap.cc:654] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.287945   794 tablet_service.cc:1468] Processing CreateTablet for tablet adb7506b712243fe91bccc326d39cdd7 (DEFAULT_TABLE table=default.Table_481f2aef57d748fa98e48e26b106b1f5 [id=0335cd54e0504a65ad54d973365cd75b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.289574  1101 tablet_service.cc:1468] Processing CreateTablet for tablet adb7506b712243fe91bccc326d39cdd7 (DEFAULT_TABLE table=default.Table_481f2aef57d748fa98e48e26b106b1f5 [id=0335cd54e0504a65ad54d973365cd75b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.296192   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet adb7506b712243fe91bccc326d39cdd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.299764  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet adb7506b712243fe91bccc326d39cdd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.236618   963 tablet_service.cc:1468] Processing CreateTablet for tablet 35fcce92db9a4e4a81f9d9db9d6283b8 (DEFAULT_TABLE table=default.Table_1cf78a5cb3c5490bae2317600c643635 [id=2613a92dda0d4236b4a4422cf612a6c6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.241187   965 tablet_service.cc:1468] Processing CreateTablet for tablet adb7506b712243fe91bccc326d39cdd7 (DEFAULT_TABLE table=default.Table_481f2aef57d748fa98e48e26b106b1f5 [id=0335cd54e0504a65ad54d973365cd75b]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.302701   957 tablet_service.cc:1468] Processing CreateTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.309412   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88d4a84c8ed7475c9bf043aaba8ad624. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.310164   798 tablet_service.cc:1468] Processing CreateTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.311146   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88d4a84c8ed7475c9bf043aaba8ad624. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.313786   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35fcce92db9a4e4a81f9d9db9d6283b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.316769  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.317250   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet adb7506b712243fe91bccc326d39cdd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.323586   961 tablet_service.cc:1468] Processing CreateTablet for tablet 5393c4c8cac1492da21b3e366aa8eaf7 (DEFAULT_TABLE table=default.Table_f94575e525fe42959d7f99555b557ead [id=8ccc90997f7a4528ae77013e11610918]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.324532   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5393c4c8cac1492da21b3e366aa8eaf7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.326316   793 tablet_service.cc:1468] Processing CreateTablet for tablet 5393c4c8cac1492da21b3e366aa8eaf7 (DEFAULT_TABLE table=default.Table_f94575e525fe42959d7f99555b557ead [id=8ccc90997f7a4528ae77013e11610918]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.327314   793 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5393c4c8cac1492da21b3e366aa8eaf7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.335131  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 5393c4c8cac1492da21b3e366aa8eaf7 (DEFAULT_TABLE table=default.Table_f94575e525fe42959d7f99555b557ead [id=8ccc90997f7a4528ae77013e11610918]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.336090   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:37.341838   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:37.344274   958 tablet_service.cc:1468] Processing CreateTablet for tablet e536268f648d4b0da1b9edc338ae974a (DEFAULT_TABLE table=default.Table_dea57d0c6e4149b1a063b0ec3b5bc013 [id=07c83477bc9c43caa67effd9d71477b4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.345352   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e536268f648d4b0da1b9edc338ae974a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.349749   798 tablet_service.cc:1468] Processing CreateTablet for tablet e536268f648d4b0da1b9edc338ae974a (DEFAULT_TABLE table=default.Table_dea57d0c6e4149b1a063b0ec3b5bc013 [id=07c83477bc9c43caa67effd9d71477b4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.349927   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:37.350741   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e536268f648d4b0da1b9edc338ae974a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.350785   540 catalog_manager.cc:5582] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:37.351943   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:37.354677  1230 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:37.355072  1230 ts_tablet_manager.cc:1397] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.121s	user 0.009s	sys 0.000s
W20250628 07:52:37.356889   758 consensus_peers.cc:489] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:37.368326   956 tablet_service.cc:1468] Processing CreateTablet for tablet 0c4a31f42b454a0b9dfac82be51563a8 (DEFAULT_TABLE table=default.Table_a5aaafa5f2c84037b58a9040e19581dd [id=390ca1dfaf6a44b193876507a9dbe929]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.369501   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c4a31f42b454a0b9dfac82be51563a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.369755  1109 tablet_service.cc:1468] Processing CreateTablet for tablet e536268f648d4b0da1b9edc338ae974a (DEFAULT_TABLE table=default.Table_dea57d0c6e4149b1a063b0ec3b5bc013 [id=07c83477bc9c43caa67effd9d71477b4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.370425   800 tablet_service.cc:1468] Processing CreateTablet for tablet 0c4a31f42b454a0b9dfac82be51563a8 (DEFAULT_TABLE table=default.Table_a5aaafa5f2c84037b58a9040e19581dd [id=390ca1dfaf6a44b193876507a9dbe929]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.371354   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c4a31f42b454a0b9dfac82be51563a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.374758  2337 consensus_queue.cc:1035] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:37.384037  2237 ts_tablet_manager.cc:1918] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.385319  2237 log.cc:1199] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:37.387192  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 0c4a31f42b454a0b9dfac82be51563a8 (DEFAULT_TABLE table=default.Table_a5aaafa5f2c84037b58a9040e19581dd [id=390ca1dfaf6a44b193876507a9dbe929]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.392014  2237 ts_tablet_manager.cc:1939] T ee03996105074308bda56d36ba3a1c5d P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:37.392721  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 325) took 2958 ms. Trace:
I20250628 07:52:37.408191  2237 rpcz_store.cc:276] 0628 07:52:34.433997 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.434092 (+    95us) service_pool.cc:225] Handling call
0628 07:52:37.229904 (+2795812us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:37.229948 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.326242 (+ 96294us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.373896 (+ 47654us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.392676 (+ 18780us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2795158}
I20250628 07:52:37.409509   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet ee03996105074308bda56d36ba3a1c5d (table default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]) successfully deleted
I20250628 07:52:37.416923  2237 tablet_replica.cc:331] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:37.417750  2237 raft_consensus.cc:2241] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:37.418643  2237 raft_consensus.cc:2270] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.419165  2364 raft_consensus.cc:491] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.419647  2364 raft_consensus.cc:513] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.420164  2162 raft_consensus.cc:491] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.420583  2162 raft_consensus.cc:513] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.421382  2364 leader_election.cc:290] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:37.422214  2162 leader_election.cc:290] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:37.423509  1230 raft_consensus.cc:357] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.423688   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:37.424104  1230 raft_consensus.cc:383] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.424331  1230 raft_consensus.cc:738] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.426687  2257 ts_tablet_manager.cc:1918] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.427069  2257 log.cc:1199] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/5fa4299a9e0f4b409d04c8c5900901c0
I20250628 07:52:37.427759  2257 ts_tablet_manager.cc:1939] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:37.388736   540 catalog_manager.cc:5582] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:37.428378   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:37.428794   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]). Expected version 1 got 0
I20250628 07:52:37.429265  2237 ts_tablet_manager.cc:1905] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.429473   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
W20250628 07:52:37.429694  1062 leader_election.cc:343] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:37.431160   540 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
W20250628 07:52:37.430310  1062 leader_election.cc:343] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:37.434077  1928 consensus_queue.cc:1035] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:37.439069   794 tablet_service.cc:1468] Processing CreateTablet for tablet 70584cf82896499ab43f7282337078ef (DEFAULT_TABLE table=default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 [id=839e2998d8e642f9a4fd0f7d376cefa4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.440079   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 70584cf82896499ab43f7282337078ef. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.443780  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 70584cf82896499ab43f7282337078ef (DEFAULT_TABLE table=default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 [id=839e2998d8e642f9a4fd0f7d376cefa4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.457448  1230 consensus_queue.cc:260] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.459734  1230 ts_tablet_manager.cc:1428] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.104s	user 0.005s	sys 0.000s
I20250628 07:52:37.462431   798 tablet_service.cc:1468] Processing CreateTablet for tablet d3e486f93a67441fa2da3a72af327503 (DEFAULT_TABLE table=default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.463683   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3e486f93a67441fa2da3a72af327503. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.465319  1105 tablet_service.cc:1468] Processing CreateTablet for tablet d3e486f93a67441fa2da3a72af327503 (DEFAULT_TABLE table=default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.470954   538 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 383) took 1760 ms. Trace:
I20250628 07:52:37.478971  1234 tablet_bootstrap.cc:492] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:37.479378  1234 ts_tablet_manager.cc:1397] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.363s	user 0.022s	sys 0.019s
I20250628 07:52:37.481562  1234 raft_consensus.cc:357] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.482187  1234 raft_consensus.cc:383] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.482431  1234 raft_consensus.cc:738] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.471251   538 rpcz_store.cc:276] 0628 07:52:35.710667 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.710785 (+   118us) service_pool.cc:225] Handling call
0628 07:52:35.711675 (+   890us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:35.712390 (+   715us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:35.712914 (+   524us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:35.713126 (+   212us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:35.798586 (+ 85460us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:35.798608 (+    22us) write_op.cc:454] Released schema lock
0628 07:52:35.799768 (+  1160us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:37.470895 (+1671127us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:35.714235 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:35.714551 (+   316us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:35.714571 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:35.714581 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:35.714807 (+   226us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:35.714844 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:35.714864 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:35.714979 (+   115us) tablet.cc:699] Row locks acquired
0628 07:52:35.714989 (+    10us) write_op.cc:260] PREPARE: finished
0628 07:52:35.715079 (+    90us) write_op.cc:270] Start()
0628 07:52:35.715153 (+    74us) write_op.cc:276] Timestamp: P: 1751097155715037 usec, L: 0
0628 07:52:35.715166 (+    13us) op_driver.cc:348] REPLICATION: starting
0628 07:52:35.715565 (+   399us) log.cc:844] Serialized 405 byte log entry
0628 07:52:35.724463 (+  8898us) op_driver.cc:464] REPLICATION: finished
0628 07:52:35.724760 (+   297us) write_op.cc:301] APPLY: starting
0628 07:52:35.724827 (+    67us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:35.724915 (+    88us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:35.725010 (+    95us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:35.725249 (+   239us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:35.725371 (+   122us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:35.725408 (+    37us) write_op.cc:312] APPLY: finished
0628 07:52:35.725549 (+   141us) log.cc:844] Serialized 23 byte log entry
0628 07:52:35.725672 (+   123us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:35.725725 (+    53us) write_op.cc:454] Released schema lock
0628 07:52:35.725755 (+    30us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":165,"apply.run_cpu_time_us":1105,"apply.run_wall_time_us":1161,"num_ops":1,"prepare.queue_time_us":103,"prepare.run_cpu_time_us":1582,"prepare.run_wall_time_us":1582,"raft.queue_time_us":1024,"raft.run_cpu_time_us":627,"raft.run_wall_time_us":761,"replication_time_us":9211}]]}
I20250628 07:52:37.487803  1234 consensus_queue.cc:260] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.493384  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88d4a84c8ed7475c9bf043aaba8ad624. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.494210   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:37.495481   545 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 361) took 5531 ms. Trace:
I20250628 07:52:37.495761   545 rpcz_store.cc:276] 0628 07:52:31.964188 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.529756 (+2565568us) service_pool.cc:225] Handling call
0628 07:52:34.621861 (+ 92105us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:34.622443 (+   582us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:34.622548 (+   105us) catalog_manager.cc:2799] Locking tablets
0628 07:52:34.623067 (+   519us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:34.773019 (+149952us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:34.873564 (+100545us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:37.494120 (+2620556us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:37.495427 (+  1307us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:34.716225 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:34.716644 (+   419us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:34.716668 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:34.716679 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:34.717002 (+   323us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:34.717039 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:34.717059 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:34.733686 (+ 16627us) tablet.cc:699] Row locks acquired
0628 07:52:34.733704 (+    18us) write_op.cc:260] PREPARE: finished
0628 07:52:34.733825 (+   121us) write_op.cc:270] Start()
0628 07:52:34.733912 (+    87us) write_op.cc:276] Timestamp: P: 1751097154733778 usec, L: 0
0628 07:52:34.733931 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:34.734405 (+   474us) log.cc:844] Serialized 771 byte log entry
0628 07:52:34.762196 (+ 27791us) op_driver.cc:464] REPLICATION: finished
0628 07:52:34.768682 (+  6486us) write_op.cc:301] APPLY: starting
0628 07:52:34.768741 (+    59us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:34.768855 (+   114us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:34.768955 (+   100us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:34.769232 (+   277us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:34.769374 (+   142us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:34.769417 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:34.769606 (+   189us) log.cc:844] Serialized 29 byte log entry
0628 07:52:34.772540 (+  2934us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.772612 (+    72us) write_op.cc:454] Released schema lock
0628 07:52:34.772646 (+    34us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:34.775949 (+  3303us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:34.776001 (+    52us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":6299,"apply.run_cpu_time_us":1400,"apply.run_wall_time_us":4163,"num_ops":2,"prepare.queue_time_us":42684,"prepare.run_cpu_time_us":2237,"prepare.run_wall_time_us":59589,"raft.queue_time_us":85,"raft.run_cpu_time_us":628,"raft.run_wall_time_us":627,"replication_time_us":28158}]]}
I20250628 07:52:37.504550   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e04e4751022f4e2fbc686f49b598150e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:37.509800  1234 ts_tablet_manager.cc:1428] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.030s	user 0.006s	sys 0.001s
W20250628 07:52:37.515938   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e04e4751022f4e2fbc686f49b598150e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:37.517586  1234 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:37.525030  2237 ts_tablet_manager.cc:1918] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.532161   800 tablet_service.cc:1515] Processing DeleteTablet for tablet e15d063a4bca40d480cba08733ec1ec3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.541522  2237 log.cc:1199] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:37.542218  2237 ts_tablet_manager.cc:1939] T cc0bf0edebb140209531ccf917fd1ed8 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:37.545660   539 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 386) took 1768 ms. Trace:
I20250628 07:52:37.547631   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ea4026116e0e482e84f4f48211e0079c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:37.545961   539 rpcz_store.cc:276] 0628 07:52:35.776825 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.786651 (+  9826us) service_pool.cc:225] Handling call
0628 07:52:35.789564 (+  2913us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:35.870083 (+ 80519us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:35.870646 (+   563us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:35.870845 (+   199us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:35.891808 (+ 20963us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:37.545606 (+1653798us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:35.876687 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:35.877036 (+   349us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:35.877055 (+    19us) write_op.cc:435] Acquired schema lock
0628 07:52:35.877066 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:35.877294 (+   228us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:35.877327 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:35.877347 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:35.877460 (+   113us) tablet.cc:699] Row locks acquired
0628 07:52:35.877471 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:35.877557 (+    86us) write_op.cc:270] Start()
0628 07:52:35.877622 (+    65us) write_op.cc:276] Timestamp: P: 1751097155877499 usec, L: 0
0628 07:52:35.877636 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:35.878032 (+   396us) log.cc:844] Serialized 405 byte log entry
0628 07:52:35.885573 (+  7541us) op_driver.cc:464] REPLICATION: finished
0628 07:52:35.888146 (+  2573us) write_op.cc:301] APPLY: starting
0628 07:52:35.888195 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:35.888274 (+    79us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:35.888364 (+    90us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:35.888504 (+   140us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:35.888612 (+   108us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:35.888651 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:35.888809 (+   158us) log.cc:844] Serialized 23 byte log entry
0628 07:52:35.888956 (+   147us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:35.889102 (+   146us) write_op.cc:454] Released schema lock
0628 07:52:35.889137 (+    35us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:35.898347 (+  9210us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:35.898372 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":1964,"child_traces":[["op",{"apply.queue_time_us":2448,"apply.run_cpu_time_us":1114,"apply.run_wall_time_us":1177,"num_ops":1,"prepare.queue_time_us":4807,"prepare.run_cpu_time_us":1646,"prepare.run_wall_time_us":21664,"raft.queue_time_us":162,"raft.run_cpu_time_us":555,"raft.run_wall_time_us":555,"replication_time_us":7836}]]}
W20250628 07:52:37.548986   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ea4026116e0e482e84f4f48211e0079c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:37.553046  1229 tablet_bootstrap.cc:492] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:37.553462  1229 ts_tablet_manager.cc:1397] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.478s	user 0.016s	sys 0.010s
I20250628 07:52:37.554412   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d3fc569c0e854f788cd8355607639d3a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:37.555720  1229 raft_consensus.cc:357] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:37.556171   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d3fc569c0e854f788cd8355607639d3a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:37.556326  1229 raft_consensus.cc:383] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.556563  1229 raft_consensus.cc:738] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.559053   798 tablet_service.cc:1468] Processing CreateTablet for tablet 1efec32bd8cd468eb13825d51722f7cc (DEFAULT_TABLE table=default.Table_786e13bb34124bc0bb4f5dae8326f725 [id=127fda336df94c48a181d1a59e0d2f00]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.560024   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1efec32bd8cd468eb13825d51722f7cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.566501   797 tablet_service.cc:1468] Processing CreateTablet for tablet 9045c6953fb64629a2aeb59b498f6d9a (DEFAULT_TABLE table=default.Table_7631c6a0720642949d4b63e271a92e4c [id=46657c237cc041279dbb9e365d574c01]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.567410   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9045c6953fb64629a2aeb59b498f6d9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.571576  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 326) took 3108 ms. Trace:
I20250628 07:52:37.571913  2237 rpcz_store.cc:276] 0628 07:52:34.435032 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.443257 (+  8225us) service_pool.cc:225] Handling call
0628 07:52:37.416777 (+2973520us) ts_tablet_manager.cc:1145] Deleting tablet cc0bf0edebb140209531ccf917fd1ed8
0628 07:52:37.416815 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.463491 (+ 46676us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.524809 (+ 61318us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.542895 (+ 18086us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2972989}
I20250628 07:52:37.573050  1234 tablet_bootstrap.cc:654] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.572978   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet cc0bf0edebb140209531ccf917fd1ed8 (table default.Table_9fc4e9c7ab7d413cba8e997e53c22844 [id=37d544e9900d4ba6912289e294a37a97]) successfully deleted
I20250628 07:52:37.576251  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5393c4c8cac1492da21b3e366aa8eaf7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.576953  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e536268f648d4b0da1b9edc338ae974a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.577615  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c4a31f42b454a0b9dfac82be51563a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.587107   800 tablet_service.cc:1468] Processing CreateTablet for tablet bf88267cb80545a5b4b2185732137576 (DEFAULT_TABLE table=default.Table_a90d3fd61b424016a14e3add816d2218 [id=57ab80d617474a998f26259768722a07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.597680  2237 tablet_replica.cc:331] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:37.598475  2237 raft_consensus.cc:2241] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:37.599395  2237 raft_consensus.cc:2270] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.601663  1234 tablet_bootstrap.cc:492] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:37.602027  1234 ts_tablet_manager.cc:1397] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.085s	user 0.007s	sys 0.002s
I20250628 07:52:37.602298  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet e15d063a4bca40d480cba08733ec1ec3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:37.602873  1175 heartbeater.cc:344] Connected to a master server at 127.0.107.189:33023
I20250628 07:52:37.603943   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:37.604056  1234 raft_consensus.cc:357] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.604605  1234 raft_consensus.cc:383] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.604562   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:37.604820  1234 raft_consensus.cc:738] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.605211  1098 tablet_service.cc:1468] Processing CreateTablet for tablet 1efec32bd8cd468eb13825d51722f7cc (DEFAULT_TABLE table=default.Table_786e13bb34124bc0bb4f5dae8326f725 [id=127fda336df94c48a181d1a59e0d2f00]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.605901  2237 ts_tablet_manager.cc:1905] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.606372  1098 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1efec32bd8cd468eb13825d51722f7cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.613631  1097 tablet_service.cc:1468] Processing CreateTablet for tablet 9045c6953fb64629a2aeb59b498f6d9a (DEFAULT_TABLE table=default.Table_7631c6a0720642949d4b63e271a92e4c [id=46657c237cc041279dbb9e365d574c01]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.614602  1097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9045c6953fb64629a2aeb59b498f6d9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.614218  1234 consensus_queue.cc:260] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.617223   798 tablet_service.cc:1468] Processing CreateTablet for tablet 5f5d8a873a9f4197b9d5304ae08c12f3 (DEFAULT_TABLE table=default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf [id=222109d2472a4f079bc6e4abf9423ee6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.617996  1229 consensus_queue.cc:260] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.620431  1229 ts_tablet_manager.cc:1428] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.067s	user 0.005s	sys 0.000s
I20250628 07:52:37.623095   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bf88267cb80545a5b4b2185732137576. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.624157   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5d8a873a9f4197b9d5304ae08c12f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.626083   796 tablet_service.cc:1468] Processing CreateTablet for tablet 0bdb4c499be441379663830f26d80204 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.627053   796 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0bdb4c499be441379663830f26d80204. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.628301   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]). Expected version 1 got 0
I20250628 07:52:37.628834   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:37.629324   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:37.630334  1229 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:37.639215  1229 tablet_bootstrap.cc:654] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.639369  1234 ts_tablet_manager.cc:1428] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.037s	user 0.005s	sys 0.000s
I20250628 07:52:37.640297  1234 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:37.640798  1096 tablet_service.cc:1468] Processing CreateTablet for tablet 5f5d8a873a9f4197b9d5304ae08c12f3 (DEFAULT_TABLE table=default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf [id=222109d2472a4f079bc6e4abf9423ee6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.643393  1096 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5d8a873a9f4197b9d5304ae08c12f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.645052   805 tablet_service.cc:1468] Processing CreateTablet for tablet d2578fbf2e974cc0bca65e32f45694ab (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.640236   538 catalog_manager.cc:5582] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:37.646472   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d2578fbf2e974cc0bca65e32f45694ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.646493   538 catalog_manager.cc:5582] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:37.647571   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:37.648188   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:37.655951   797 tablet_service.cc:1468] Processing CreateTablet for tablet c070c23e87594bf5853b2e42aafac7f5 (DEFAULT_TABLE table=default.Table_0b4dcd93c24b4479ad87780460b6afd8 [id=4fe25f3f36eb4a76addfc9f54bb85371]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.656875   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c070c23e87594bf5853b2e42aafac7f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.661248   804 tablet_service.cc:1468] Processing CreateTablet for tablet aec37016d1674945b399cfc2f92b08fc (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.662191   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aec37016d1674945b399cfc2f92b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.674670   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31] alter complete (version 3)
I20250628 07:52:37.696655   539 master_service.cc:948] ReplaceTablet: received request to replace tablet 9ba2a10acdf0462bb05628351cdbbbe4 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:37.697468  1234 tablet_bootstrap.cc:654] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.702867   874 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:37.714556  2237 ts_tablet_manager.cc:1918] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.714928  2237 log.cc:1199] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:37.715648  2237 ts_tablet_manager.cc:1939] T 83ffa65248fc4db3aecc5f91b648b83f P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:37.715724   794 tablet_service.cc:1468] Processing CreateTablet for tablet e78cb8844a43404f9ee38dec2e532169 (DEFAULT_TABLE table=default.Table_b46689c124f54809961340f6c6b317d5 [id=bc2345b772a947e696627993d69e5e02]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.716423  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 327) took 3271 ms. Trace:
I20250628 07:52:37.717262   545 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 397) took 1111 ms. Trace:
I20250628 07:52:37.717550   545 rpcz_store.cc:276] 0628 07:52:36.605393 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.547304 (+941911us) service_pool.cc:225] Handling call
0628 07:52:37.549388 (+  2084us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:37.588385 (+ 38997us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:37.716685 (+128300us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:37.716965 (+   280us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:37.717209 (+   244us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:37.589689 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:37.590036 (+   347us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:37.590058 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:37.590072 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:37.590324 (+   252us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:37.590358 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:37.590379 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:37.590585 (+   206us) tablet.cc:699] Row locks acquired
0628 07:52:37.590598 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:37.590697 (+    99us) write_op.cc:270] Start()
0628 07:52:37.590770 (+    73us) write_op.cc:276] Timestamp: P: 1751097157590620 usec, L: 0
0628 07:52:37.590785 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:37.591125 (+   340us) log.cc:844] Serialized 433 byte log entry
0628 07:52:37.713912 (+122787us) op_driver.cc:464] REPLICATION: finished
0628 07:52:37.715311 (+  1399us) write_op.cc:301] APPLY: starting
0628 07:52:37.715361 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:37.715458 (+    97us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:37.715556 (+    98us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:37.715948 (+   392us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:37.716074 (+   126us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:37.716116 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:37.716276 (+   160us) log.cc:844] Serialized 29 byte log entry
0628 07:52:37.716406 (+   130us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:37.716475 (+    69us) write_op.cc:454] Released schema lock
0628 07:52:37.716506 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:37.716725 (+   219us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:37.716747 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1239,"apply.run_cpu_time_us":1378,"apply.run_wall_time_us":1397,"num_ops":2,"prepare.queue_time_us":91,"prepare.run_cpu_time_us":1581,"prepare.run_wall_time_us":1581,"replication_time_us":123037}]]}
I20250628 07:52:37.714135  2162 raft_consensus.cc:491] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.721515  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 321) took 1551 ms. Trace:
I20250628 07:52:37.721565  2162 raft_consensus.cc:513] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.721931  2257 rpcz_store.cc:276] 0628 07:52:36.170282 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.170446 (+   164us) service_pool.cc:225] Handling call
0628 07:52:37.155349 (+984903us) ts_tablet_manager.cc:1145] Deleting tablet 5fa4299a9e0f4b409d04c8c5900901c0
0628 07:52:37.155389 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.318555 (+163166us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.426420 (+107865us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.721451 (+295031us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":984125}
I20250628 07:52:37.723201  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 322) took 1221 ms. Trace:
I20250628 07:52:37.723366  2162 leader_election.cc:290] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:37.723834  2162 raft_consensus.cc:491] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.725425  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 70584cf82896499ab43f7282337078ef. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.725797  1230 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:37.723501  2257 rpcz_store.cc:276] 0628 07:52:36.502029 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.502796 (+   767us) service_pool.cc:225] Handling call
0628 07:52:37.722852 (+1220056us) ts_tablet_manager.cc:1145] Deleting tablet af28ae2139b2417381808cb4068ddf60
0628 07:52:37.722891 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.723145 (+   254us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1200995}
I20250628 07:52:37.729903   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:37.731175  1234 tablet_bootstrap.cc:492] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
W20250628 07:52:37.731151  1062 leader_election.cc:343] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:37.731544  1234 ts_tablet_manager.cc:1397] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.091s	user 0.008s	sys 0.000s
I20250628 07:52:37.720656   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4ee15e08b0884908a81ad107f6738fe8"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:37.732918   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4ee15e08b0884908a81ad107f6738fe8 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:37.733619  1234 raft_consensus.cc:357] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.734191  1234 raft_consensus.cc:383] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.734409  1234 raft_consensus.cc:738] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.716730  2237 rpcz_store.cc:276] 0628 07:52:34.444653 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.444746 (+    93us) service_pool.cc:225] Handling call
0628 07:52:37.575424 (+3130678us) ts_tablet_manager.cc:1145] Deleting tablet 83ffa65248fc4db3aecc5f91b648b83f
0628 07:52:37.597524 (+ 22100us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.648694 (+ 51170us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.714344 (+ 65650us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.716374 (+  2030us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":2816,"tablet-delete.queue_time_us":3130109}
I20250628 07:52:37.724201  2162 raft_consensus.cc:513] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.739244   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 83ffa65248fc4db3aecc5f91b648b83f (table default.Table_8c6a7fbba6c24b8181422482c95fe4b7 [id=7b63b959f4874031ae5ba6f8b61a4135]) successfully deleted
I20250628 07:52:37.740547   793 tablet_service.cc:1515] Processing DeleteTablet for tablet 0dffb3eb778946869b914e5b05a67a69 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:22 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.740820   800 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.741127   803 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.741389   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 9579dd44e5e241aea2671be28e16ef7c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.741182  1234 consensus_queue.cc:260] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.741951   798 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.733338  1106 tablet_service.cc:1468] Processing CreateTablet for tablet d2578fbf2e974cc0bca65e32f45694ab (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.746949   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.747264   800 tablet_service.cc:1515] Processing DeleteTablet for tablet 5ec8c1d9386a4922b55df5e06039858d with delete_type TABLET_DATA_DELETED (replaced by tablet 95fa7c0a68d34d728042fb5a8a3b7307) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.747465   796 tablet_service.cc:1515] Processing DeleteTablet for tablet 5475a89692eb450d82cde521ec8c9a73 with delete_type TABLET_DATA_DELETED (replaced by tablet 2595308cbc0d44f0abfd0a126f16d4df) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:37.748136   798 tablet_service.cc:1515] Processing DeleteTablet for tablet b2d49e5a06384758bce5e7328b6a10f8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:41482
W20250628 07:52:37.750522   542 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 63) took 2017 ms (client timeout 1000 ms). Trace:
I20250628 07:52:37.753783  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d2578fbf2e974cc0bca65e32f45694ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.765702  1234 ts_tablet_manager.cc:1428] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.034s	user 0.007s	sys 0.000s
I20250628 07:52:37.781575   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 5fa4299a9e0f4b409d04c8c5900901c0 (table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]) successfully deleted
W20250628 07:52:37.782258   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet af28ae2139b2417381808cb4068ddf60 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet af28ae2139b2417381808cb4068ddf60 already in progress: creating tablet
W20250628 07:52:37.783799   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet f0c00063b47e49099a2c0ab9c8587c04 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f0c00063b47e49099a2c0ab9c8587c04 already in progress: creating tablet
I20250628 07:52:37.787111  2237 tablet_replica.cc:331] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:37.788089  2237 raft_consensus.cc:2241] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:37.789177  2237 raft_consensus.cc:2270] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.792423  2237 ts_tablet_manager.cc:1905] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.793694  1234 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:37.764195  2257 tablet_replica.cc:331] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:37.800103  2257 raft_consensus.cc:2241] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:37.784636  1095 tablet_service.cc:1468] Processing CreateTablet for tablet 0bdb4c499be441379663830f26d80204 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.804230  2257 raft_consensus.cc:2270] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.804510  1234 tablet_bootstrap.cc:654] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.808961  1099 tablet_service.cc:1468] Processing CreateTablet for tablet bf88267cb80545a5b4b2185732137576 (DEFAULT_TABLE table=default.Table_a90d3fd61b424016a14e3add816d2218 [id=57ab80d617474a998f26259768722a07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.810217  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3e486f93a67441fa2da3a72af327503. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.810356   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e78cb8844a43404f9ee38dec2e532169. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.811704  1099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bf88267cb80545a5b4b2185732137576. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.811976  1230 tablet_bootstrap.cc:654] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:37.819137  1094 tablet_service.cc:1468] Processing CreateTablet for tablet aec37016d1674945b399cfc2f92b08fc (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.821151  1095 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0bdb4c499be441379663830f26d80204. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.839949  1229 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:37.840353  1229 ts_tablet_manager.cc:1397] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.210s	user 0.007s	sys 0.003s
I20250628 07:52:37.847148  1234 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:37.847555  1234 ts_tablet_manager.cc:1397] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.054s	user 0.006s	sys 0.002s
I20250628 07:52:37.859323  1234 raft_consensus.cc:357] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.859975  1234 raft_consensus.cc:383] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.860220  1234 raft_consensus.cc:738] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.872449  2237 ts_tablet_manager.cc:1918] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:37.872879  2237 log.cc:1199] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:37.873696  2237 ts_tablet_manager.cc:1939] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:37.860663  1229 raft_consensus.cc:357] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.876010  1229 raft_consensus.cc:383] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.876317  1229 raft_consensus.cc:738] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.877020  1229 consensus_queue.cc:260] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.880008  1229 ts_tablet_manager.cc:1428] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.039s	user 0.005s	sys 0.000s
I20250628 07:52:37.860750  1234 consensus_queue.cc:260] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.899221   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:37.900673   825 raft_consensus.cc:2466] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:37.902647  1062 leader_election.cc:304] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:37.905520  2364 raft_consensus.cc:2802] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:37.905836  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aec37016d1674945b399cfc2f92b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:37.875118  1103 tablet_service.cc:1468] Processing CreateTablet for tablet c070c23e87594bf5853b2e42aafac7f5 (DEFAULT_TABLE table=default.Table_0b4dcd93c24b4479ad87780460b6afd8 [id=4fe25f3f36eb4a76addfc9f54bb85371]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:37.905874  2364 raft_consensus.cc:491] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:37.907800  2364 raft_consensus.cc:3058] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:37.913172  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 328) took 3467 ms. Trace:
I20250628 07:52:37.917259  2237 rpcz_store.cc:276] 0628 07:52:34.445663 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.445771 (+   108us) service_pool.cc:225] Handling call
0628 07:52:37.735107 (+3289336us) ts_tablet_manager.cc:1145] Deleting tablet 46ff641001cd4b01a4c09c0d8b79ebf3
0628 07:52:37.786967 (+ 51860us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:37.813984 (+ 27017us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.872183 (+ 58199us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:37.913109 (+ 40926us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3288401}
I20250628 07:52:37.920475  2237 tablet_replica.cc:331] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:37.926069  2237 raft_consensus.cc:2241] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:37.912361  1234 ts_tablet_manager.cc:1428] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.065s	user 0.006s	sys 0.001s
I20250628 07:52:37.930200  2237 raft_consensus.cc:2270] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:37.935926  1230 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:37.936306  1230 ts_tablet_manager.cc:1397] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.211s	user 0.012s	sys 0.004s
I20250628 07:52:37.938503  1230 raft_consensus.cc:357] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.939878  1230 raft_consensus.cc:383] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:37.940261  1230 raft_consensus.cc:738] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:37.940881  1230 consensus_queue.cc:260] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.942204  2162 leader_election.cc:290] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:37.942766  2257 ts_tablet_manager.cc:1905] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.948848   584 master_runner.cc:186] Error getting master registration for 127.0.107.190:37209: OK, instance_id { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" instance_seqno: 1751097153378333 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20250628 07:52:37.949651   584 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:52:37.952829  2175 raft_consensus.cc:491] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:37.953210  2175 raft_consensus.cc:513] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:37.955639  2175 leader_election.cc:290] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:37.963403  2237 ts_tablet_manager.cc:1905] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:37.973906  1234 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:37.750934   542 rpcz_store.cc:269] 0628 07:52:35.732984 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.733148 (+   164us) service_pool.cc:225] Handling call
0628 07:52:37.750431 (+2017283us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:36.801142 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:36.801494 (+   352us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:36.801516 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:36.801527 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:36.801970 (+   443us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:36.802005 (+    35us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:36.802024 (+    19us) tablet.cc:683] Acquiring locks for 4 operations
0628 07:52:36.802288 (+   264us) tablet.cc:699] Row locks acquired
0628 07:52:36.802300 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:36.802391 (+    91us) write_op.cc:270] Start()
0628 07:52:36.802455 (+    64us) write_op.cc:276] Timestamp: P: 1751097156802339 usec, L: 0
0628 07:52:36.802468 (+    13us) op_driver.cc:348] REPLICATION: starting
0628 07:52:36.802917 (+   449us) log.cc:844] Serialized 1674 byte log entry
0628 07:52:36.811996 (+  9079us) op_driver.cc:464] REPLICATION: finished
0628 07:52:36.812285 (+   289us) write_op.cc:301] APPLY: starting
0628 07:52:36.812338 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:36.812483 (+   145us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:36.812578 (+    95us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:36.813016 (+   438us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:36.813218 (+   202us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=4
0628 07:52:36.813258 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:36.813456 (+   198us) log.cc:844] Serialized 41 byte log entry
0628 07:52:36.813594 (+   138us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.813682 (+    88us) write_op.cc:454] Released schema lock
0628 07:52:36.813733 (+    51us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:36.813971 (+   238us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.813994 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":53302,"reported_tablets":36,"child_traces":[["op",{"apply.queue_time_us":159,"apply.run_cpu_time_us":1663,"apply.run_wall_time_us":1660,"num_ops":4,"prepare.queue_time_us":172,"prepare.run_cpu_time_us":2014,"prepare.run_wall_time_us":2015,"raft.queue_time_us":192,"raft.run_cpu_time_us":541,"raft.run_wall_time_us":539,"replication_time_us":9450}]]}
I20250628 07:52:37.987695   542 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9f45c9f9b24e4c9782068f570ce3f3e7" } modify_external_catalogs: true
I20250628 07:52:37.988276   542 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9f45c9f9b24e4c9782068f570ce3f3e7" } modify_external_catalogs: true
W20250628 07:52:37.989915   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 62) took 3330 ms (client timeout 1000 ms). Trace:
W20250628 07:52:37.990272   546 rpcz_store.cc:269] 0628 07:52:34.659710 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.689304 (+ 29594us) service_pool.cc:225] Handling call
0628 07:52:37.989855 (+3300551us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":47400,"reported_tablets":37}
I20250628 07:52:37.991128   546 master_service.cc:948] ReplaceTablet: received request to replace tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:37.992781   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (table default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]) successfully deleted
I20250628 07:52:38.010921  1234 tablet_bootstrap.cc:654] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:38.006948   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 72) took 1582 ms (client timeout 1000 ms). Trace:
W20250628 07:52:38.014801   544 rpcz_store.cc:269] 0628 07:52:36.424456 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.501964 (+ 77508us) service_pool.cc:225] Handling call
0628 07:52:38.006879 (+1504915us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":296478,"reported_tablets":40}
I20250628 07:52:38.017515   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_e1df74fb3dfa455bb3300ad20e7a2268" } new_table_name: "default.Table_cb9c785de9174bbd909ba92bb4377d46" modify_external_catalogs: true
I20250628 07:52:38.022851  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:38.023550  2162 raft_consensus.cc:491] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.023990  2162 raft_consensus.cc:513] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.023523  1122 raft_consensus.cc:2391] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
W20250628 07:52:38.060037   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 71) took 2699 ms (client timeout 1000 ms). Trace:
I20250628 07:52:38.066578  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.066629  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.069286  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.070825  1101 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 with delete_type TABLET_DATA_DELETED (replaced by tablet aec37016d1674945b399cfc2f92b08fc) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.072870  1230 ts_tablet_manager.cc:1428] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.135s	user 0.012s	sys 0.004s
I20250628 07:52:38.081768  2175 raft_consensus.cc:491] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.082281  2175 raft_consensus.cc:513] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.084285  2175 leader_election.cc:290] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:38.094429  2378 log.cc:927] Time spent T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Append to log took a long time: real 0.189s	user 0.000s	sys 0.000s
I20250628 07:52:38.094558  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.095449  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.098989  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:38.099588  1129 raft_consensus.cc:2466] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:38.101312   758 leader_election.cc:304] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:38.102155  2175 raft_consensus.cc:2802] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:38.102566  2175 raft_consensus.cc:491] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:38.102953  2175 raft_consensus.cc:3058] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.104079  1100 tablet_service.cc:1515] Processing DeleteTablet for tablet b51b51eb951d45a78bd2bffe47834b95 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:23 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.104533  1093 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.106456  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c070c23e87594bf5853b2e42aafac7f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.106982  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.117247  1092 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.117954  1093 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.119892  2364 raft_consensus.cc:513] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.127256  1230 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:38.102583   541 rpcz_store.cc:269] 0628 07:52:35.360877 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.360964 (+    87us) service_pool.cc:225] Handling call
0628 07:52:38.059967 (+2699003us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":69883,"reported_tablets":40}
W20250628 07:52:38.123859   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 57) took 2173 ms (client timeout 1000 ms). Trace:
W20250628 07:52:38.134464   543 rpcz_store.cc:269] 0628 07:52:35.950223 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.952533 (+  2310us) service_pool.cc:225] Handling call
0628 07:52:38.123782 (+2171249us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:36.064279 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:36.064653 (+   374us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:36.064674 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:36.064687 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:36.064979 (+   292us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:36.065015 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:36.065038 (+    23us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:36.065292 (+   254us) tablet.cc:699] Row locks acquired
0628 07:52:36.065305 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:36.065406 (+   101us) write_op.cc:270] Start()
0628 07:52:36.065477 (+    71us) write_op.cc:276] Timestamp: P: 1751097156065330 usec, L: 0
0628 07:52:36.065492 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:36.065939 (+   447us) log.cc:844] Serialized 913 byte log entry
0628 07:52:36.108275 (+ 42336us) op_driver.cc:464] REPLICATION: finished
0628 07:52:36.108830 (+   555us) write_op.cc:301] APPLY: starting
0628 07:52:36.108883 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:36.108977 (+    94us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:36.109052 (+    75us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:36.109322 (+   270us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:36.109445 (+   123us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:36.109562 (+   117us) write_op.cc:312] APPLY: finished
0628 07:52:36.109717 (+   155us) log.cc:844] Serialized 29 byte log entry
0628 07:52:36.109843 (+   126us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.109965 (+   122us) write_op.cc:454] Released schema lock
0628 07:52:36.109994 (+    29us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:36.113319 (+  3325us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.113345 (+    26us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:36.151703 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:36.152065 (+   362us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:36.152088 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:36.152101 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:36.159208 (+  7107us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:36.159247 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:36.159269 (+    22us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:36.159404 (+   135us) tablet.cc:699] Row locks acquired
0628 07:52:36.159416 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:36.159538 (+   122us) write_op.cc:270] Start()
0628 07:52:36.159619 (+    81us) write_op.cc:276] Timestamp: P: 1751097156159486 usec, L: 0
0628 07:52:36.159635 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:36.170225 (+ 10590us) log.cc:844] Serialized 356 byte log entry
0628 07:52:36.185640 (+ 15415us) op_driver.cc:464] REPLICATION: finished
0628 07:52:36.190149 (+  4509us) write_op.cc:301] APPLY: starting
0628 07:52:36.190194 (+    45us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:36.190274 (+    80us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:36.190347 (+    73us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:36.190487 (+   140us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:36.190632 (+   145us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:36.190672 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:36.191372 (+   700us) log.cc:844] Serialized 23 byte log entry
0628 07:52:36.191502 (+   130us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.191566 (+    64us) write_op.cc:454] Released schema lock
0628 07:52:36.191592 (+    26us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:36.191822 (+   230us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:36.191851 (+    29us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":61675,"reported_tablets":13,"child_traces":[["op",{"apply.queue_time_us":422,"apply.run_cpu_time_us":1238,"apply.run_wall_time_us":4448,"num_ops":2,"prepare.queue_time_us":19965,"prepare.run_cpu_time_us":1974,"prepare.run_wall_time_us":35581,"raft.queue_time_us":104,"raft.run_cpu_time_us":687,"raft.run_wall_time_us":687,"replication_time_us":42703}],["op",{"apply.queue_time_us":4350,"apply.run_cpu_time_us":1632,"apply.run_wall_time_us":1633,"num_ops":1,"prepare.queue_time_us":36554,"prepare.run_cpu_time_us":1960,"prepare.run_wall_time_us":35856,"raft.queue_time_us":5550,"raft.run_cpu_time_us":613,"raft.run_wall_time_us":613,"replication_time_us":25924}]]}
I20250628 07:52:38.140128   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_10140fc26c314afab000f5f4626f9110" } modify_external_catalogs: true
I20250628 07:52:38.142577  1229 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:38.150269   534 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 303753us
I20250628 07:52:38.156103  2400 tablet.cc:1722] T 58e7ec1f221f41ca8f6b019faeec3377 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:38.160821  2237 ts_tablet_manager.cc:1918] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:38.132778  2175 raft_consensus.cc:513] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:38.165045   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 70) took 3907 ms (client timeout 1000 ms). Trace:
W20250628 07:52:38.165354   547 rpcz_store.cc:269] 0628 07:52:34.257616 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.342189 (+ 84573us) service_pool.cc:225] Handling call
0628 07:52:37.690792 (+3348603us) spinlock_profiling.cc:243] Waited 6.27 ms on lock 0x7b6800041f60. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5bea863af1 00007f5bea84112a 00007f5bea83f868 00007f5bea83de90 00007f5bea9400ae 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc
0628 07:52:38.164987 (+474195us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":54592,"reported_tablets":40,"spinlock_wait_cycles":14426368}
I20250628 07:52:38.166455   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_77748d06f2a54596a43e8a89b5b56011"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.167833   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_77748d06f2a54596a43e8a89b5b56011 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:38.171765   956 consensus_meta.cc:287] T 0c4a31f42b454a0b9dfac82be51563a8 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.792s	user 0.004s	sys 0.004s
W20250628 07:52:38.176299   957 consensus_meta.cc:287] T 88d4a84c8ed7475c9bf043aaba8ad624 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.837s	user 0.002s	sys 0.007s
I20250628 07:52:38.179829  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:38.180501  1124 raft_consensus.cc:3058] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.182561  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.190666  1234 tablet_bootstrap.cc:492] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:38.191079  1234 ts_tablet_manager.cc:1397] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.217s	user 0.010s	sys 0.007s
W20250628 07:52:38.201977   955 consensus_meta.cc:287] T f91208e5de0b42a5a3e17f26f423ea03 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.792s	user 0.004s	sys 0.004s
I20250628 07:52:38.205353  1229 tablet_bootstrap.cc:654] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.214825  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:38.214794  2369 lock_manager.cc:585] Waited 1 seconds to obtain row lock on key '<redacted>' (key hash 15122197062527616531) tablet 00000000000000000000000000000000 cur holder 0x7b5800054600
I20250628 07:52:38.215803  2175 leader_election.cc:290] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:38.216909  1100 tablet_service.cc:1468] Processing CreateTablet for tablet e78cb8844a43404f9ee38dec2e532169 (DEFAULT_TABLE table=default.Table_b46689c124f54809961340f6c6b317d5 [id=bc2345b772a947e696627993d69e5e02]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.218060  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e78cb8844a43404f9ee38dec2e532169. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.221103   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:38.221662   825 raft_consensus.cc:3058] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.227618   825 raft_consensus.cc:2466] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:38.228575  1062 leader_election.cc:304] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:38.248481  2389 raft_consensus.cc:491] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.252929  2162 leader_election.cc:290] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:38.253384  2162 raft_consensus.cc:491] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.253739  2162 raft_consensus.cc:513] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.255748  2162 leader_election.cc:290] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:38.256248  2162 raft_consensus.cc:2802] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:38.256687  2162 raft_consensus.cc:695] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:38.219542  2364 leader_election.cc:290] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:38.259338  1229 tablet_bootstrap.cc:492] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:38.259753  1229 ts_tablet_manager.cc:1397] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.379s	user 0.007s	sys 0.006s
I20250628 07:52:38.247782  1092 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.248907  1093 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.270929   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:38.271301   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_10140fc26c314afab000f5f4626f9110" } modify_external_catalogs: true
I20250628 07:52:38.272256   545 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 400) took 1238 ms. Trace:
I20250628 07:52:38.273356   539 catalog_manager.cc:6652] ReplaceTablet: tablet 9ba2a10acdf0462bb05628351cdbbbe4 deleted and replaced by tablet 80ee935a2d58422fb1bfbc9103940920
I20250628 07:52:38.273694   539 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 399) took 1509 ms. Trace:
I20250628 07:52:38.274443   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 with delete_type TABLET_DATA_DELETED (replaced by tablet 80ee935a2d58422fb1bfbc9103940920) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:38.272589   545 rpcz_store.cc:276] 0628 07:52:37.034092 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.720334 (+686242us) service_pool.cc:225] Handling call
0628 07:52:37.733318 (+ 12984us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:38.161531 (+428213us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:38.210248 (+ 48717us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:38.271982 (+ 61734us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:38.272207 (+   225us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:38.162969 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:38.163274 (+   305us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:38.163311 (+    37us) write_op.cc:435] Acquired schema lock
0628 07:52:38.163322 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:38.163569 (+   247us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:38.163600 (+    31us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:38.163623 (+    23us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:38.163807 (+   184us) tablet.cc:699] Row locks acquired
0628 07:52:38.163819 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:38.163909 (+    90us) write_op.cc:270] Start()
0628 07:52:38.163973 (+    64us) write_op.cc:276] Timestamp: P: 1751097158163829 usec, L: 0
0628 07:52:38.163990 (+    17us) op_driver.cc:348] REPLICATION: starting
0628 07:52:38.164405 (+   415us) log.cc:844] Serialized 433 byte log entry
0628 07:52:38.205842 (+ 41437us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.207974 (+  2132us) write_op.cc:301] APPLY: starting
0628 07:52:38.208013 (+    39us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.208111 (+    98us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.208209 (+    98us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.208618 (+   409us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.208766 (+   148us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:38.208820 (+    54us) write_op.cc:312] APPLY: finished
0628 07:52:38.208974 (+   154us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.209079 (+   105us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.210030 (+   951us) write_op.cc:454] Released schema lock
0628 07:52:38.210064 (+    34us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2017,"apply.run_cpu_time_us":1423,"apply.run_wall_time_us":2264,"num_ops":2,"prepare.queue_time_us":457,"prepare.run_cpu_time_us":1616,"prepare.run_wall_time_us":1616,"replication_time_us":41778,"thread_start_us":56302,"threads_started":1}]]}
I20250628 07:52:38.276362  2237 log.cc:1199] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/1c65fa809b17418c85d4d72ae1e0ee3a
I20250628 07:52:38.273989   539 rpcz_store.cc:276] 0628 07:52:36.763832 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.696559 (+932727us) service_pool.cc:225] Handling call
0628 07:52:38.273646 (+577087us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:37.701632 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:37.701955 (+   323us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:37.701975 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:37.701986 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:37.702242 (+   256us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:37.702272 (+    30us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:37.702290 (+    18us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:37.702447 (+   157us) tablet.cc:699] Row locks acquired
0628 07:52:37.702476 (+    29us) write_op.cc:260] PREPARE: finished
0628 07:52:37.702570 (+    94us) write_op.cc:270] Start()
0628 07:52:37.702647 (+    77us) write_op.cc:276] Timestamp: P: 1751097157702537 usec, L: 0
0628 07:52:37.702659 (+    12us) op_driver.cc:348] REPLICATION: starting
0628 07:52:37.703196 (+   537us) log.cc:844] Serialized 662 byte log entry
0628 07:52:38.145742 (+442546us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.146040 (+   298us) write_op.cc:301] APPLY: starting
0628 07:52:38.146092 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.146183 (+    91us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.146261 (+    78us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.146570 (+   309us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.146739 (+   169us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:38.146776 (+    37us) write_op.cc:312] APPLY: finished
0628 07:52:38.146950 (+   174us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.147086 (+   136us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.147138 (+    52us) write_op.cc:454] Released schema lock
0628 07:52:38.147163 (+    25us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:38.147336 (+   173us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.147361 (+    25us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":163,"apply.run_cpu_time_us":1273,"apply.run_wall_time_us":1273,"num_ops":2,"prepare.queue_time_us":173,"prepare.run_cpu_time_us":1794,"prepare.run_wall_time_us":1794,"replication_time_us":442991}]]}
I20250628 07:52:38.277637   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d86e7bc180df48c0b8358485d2a8bc2f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:38.279590   542 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet 9dd50571d8394da7b636808307047d5e
I20250628 07:52:38.279948   542 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 401) took 1164 ms. Trace:
I20250628 07:52:38.280535  2337 raft_consensus.cc:491] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.281214  2337 raft_consensus.cc:513] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.280404   542 rpcz_store.cc:276] 0628 07:52:37.115329 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.987417 (+872088us) service_pool.cc:225] Handling call
0628 07:52:37.988706 (+  1289us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:38.145636 (+156930us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:38.145775 (+   139us) catalog_manager.cc:2799] Locking tablets
0628 07:52:38.146219 (+   444us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:38.254343 (+108124us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:38.279407 (+ 25064us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:38.279514 (+   107us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:38.279896 (+   382us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:38.154056 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:38.154389 (+   333us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:38.154410 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:38.154425 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:38.154695 (+   270us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:38.154725 (+    30us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:38.154747 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:38.154925 (+   178us) tablet.cc:699] Row locks acquired
0628 07:52:38.154936 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:38.155032 (+    96us) write_op.cc:270] Start()
0628 07:52:38.155101 (+    69us) write_op.cc:276] Timestamp: P: 1751097158154949 usec, L: 0
0628 07:52:38.155115 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:38.155521 (+   406us) log.cc:844] Serialized 533 byte log entry
0628 07:52:38.205620 (+ 50099us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.206275 (+   655us) write_op.cc:301] APPLY: starting
0628 07:52:38.206335 (+    60us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.206449 (+   114us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.206546 (+    97us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.206808 (+   262us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.206969 (+   161us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:38.207021 (+    52us) write_op.cc:312] APPLY: finished
0628 07:52:38.207222 (+   201us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.207372 (+   150us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.207435 (+    63us) write_op.cc:454] Released schema lock
0628 07:52:38.207470 (+    35us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:38.207709 (+   239us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.207738 (+    29us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":502,"apply.run_cpu_time_us":1401,"apply.run_wall_time_us":1402,"num_ops":2,"prepare.queue_time_us":6647,"prepare.run_cpu_time_us":1631,"prepare.run_wall_time_us":1631,"replication_time_us":50413}]]}
I20250628 07:52:38.283401  2337 leader_election.cc:290] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:38.292114  1230 tablet_bootstrap.cc:654] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.294466  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 with delete_type TABLET_DATA_DELETED (replaced by tablet 80ee935a2d58422fb1bfbc9103940920) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.270716  1234 raft_consensus.cc:357] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.305310  1234 raft_consensus.cc:383] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.311789   546 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 8f7e8d9640c74c8da5ac00adaf86f8a1
I20250628 07:52:38.314018   546 catalog_manager.cc:6652] ReplaceTablet: tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 deleted and replaced by tablet 586f1b7dd30e4998abebaa1fb2983e2b
I20250628 07:52:38.314301   546 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 402) took 1187 ms. Trace:
I20250628 07:52:38.314535   546 rpcz_store.cc:276] 0628 07:52:37.127180 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.991021 (+863841us) service_pool.cc:225] Handling call
0628 07:52:38.314257 (+323236us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:38.264917 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:38.268150 (+  3233us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:38.268172 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:38.268184 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:38.268463 (+   279us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:38.268497 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:38.268519 (+    22us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:38.268687 (+   168us) tablet.cc:699] Row locks acquired
0628 07:52:38.268699 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:38.268806 (+   107us) write_op.cc:270] Start()
0628 07:52:38.268879 (+    73us) write_op.cc:276] Timestamp: P: 1751097158268765 usec, L: 0
0628 07:52:38.268894 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:38.269373 (+   479us) log.cc:844] Serialized 662 byte log entry
0628 07:52:38.307077 (+ 37704us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.308969 (+  1892us) write_op.cc:301] APPLY: starting
0628 07:52:38.309015 (+    46us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.309130 (+   115us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.309236 (+   106us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.309552 (+   316us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.309770 (+   218us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:38.309812 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:38.309962 (+   150us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.310099 (+   137us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.310159 (+    60us) write_op.cc:454] Released schema lock
0628 07:52:38.310187 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:38.310368 (+   181us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.310391 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1791,"apply.run_cpu_time_us":1370,"apply.run_wall_time_us":1369,"num_ops":2,"prepare.queue_time_us":10938,"prepare.run_cpu_time_us":1816,"prepare.run_wall_time_us":5631,"replication_time_us":38122}]]}
I20250628 07:52:38.317045  1229 raft_consensus.cc:357] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.322311   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3c5a83219fbc4befa57b1095ee0e4fa0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.323709   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:38.278999   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d86e7bc180df48c0b8358485d2a8bc2f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.293534  2389 raft_consensus.cc:513] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.331354  2389 leader_election.cc:290] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:38.336728  1093 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 with delete_type TABLET_DATA_DELETED (replaced by tablet 586f1b7dd30e4998abebaa1fb2983e2b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.337590  1229 raft_consensus.cc:383] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.337874  1229 raft_consensus.cc:738] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.349033  1229 consensus_queue.cc:260] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.352255  1124 raft_consensus.cc:2466] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:38.330006  2162 consensus_queue.cc:237] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.389043  1229 ts_tablet_manager.cc:1428] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.129s	user 0.006s	sys 0.001s
I20250628 07:52:38.403519   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_6865e743213148aba2ee6d7058bc7219" } new_table_name: "default.Table_a211a91358794a51a65e048e19371379" modify_external_catalogs: true
I20250628 07:52:38.404352  1234 raft_consensus.cc:738] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.405704  1234 consensus_queue.cc:260] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.410295   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79d15e600b8b460fab54eb21193f1f40" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.411550   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.412063   984 raft_consensus.cc:2466] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:38.412906   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "63115b7be4a94b639c18601d438ed947" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:38.413473   983 raft_consensus.cc:3058] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.417080  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:38.418673  2397 raft_consensus.cc:491] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.419023  2397 raft_consensus.cc:513] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.420756  2397 leader_election.cc:290] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:38.425295  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 196787us
I20250628 07:52:38.427088   758 leader_election.cc:304] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:38.427711   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:38.427779   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:38.428737   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:38.429283  2397 raft_consensus.cc:2802] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:38.429945  2397 raft_consensus.cc:695] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:38.433074  2397 consensus_queue.cc:237] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.435030   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.435546   980 raft_consensus.cc:2466] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:38.424935   983 raft_consensus.cc:2466] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:38.277077  2237 ts_tablet_manager.cc:1939] T 1c65fa809b17418c85d4d72ae1e0ee3a P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:38.316059   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 with delete_type TABLET_DATA_DELETED (replaced by tablet 586f1b7dd30e4998abebaa1fb2983e2b) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:38.335124   818 raft_consensus.cc:3058] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.336397   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:38.340844   825 raft_consensus.cc:3058] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.437403   824 raft_consensus.cc:2391] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:38.438230  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 329) took 3777 ms. Trace:
I20250628 07:52:38.438521  2237 rpcz_store.cc:276] 0628 07:52:34.660433 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.666822 (+  6389us) service_pool.cc:225] Handling call
0628 07:52:37.920296 (+3253474us) ts_tablet_manager.cc:1145] Deleting tablet 1c65fa809b17418c85d4d72ae1e0ee3a
0628 07:52:37.920349 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:38.061062 (+140713us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.160586 (+ 99524us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.438181 (+277595us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3252919}
I20250628 07:52:38.439527  2237 tablet_replica.cc:331] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:38.443853  2257 ts_tablet_manager.cc:1918] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20250628 07:52:38.444190  2257 log.cc:1199] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:38.444833  2257 ts_tablet_manager.cc:1939] T e15d063a4bca40d480cba08733ec1ec3 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:38.446756   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 1c65fa809b17418c85d4d72ae1e0ee3a (table default.Table_c84bc632fbed42de8d7f5bc303bcaa61 [id=efbc4a9eb51f43b1a08b36566c1ab887]) successfully deleted
I20250628 07:52:38.446914  2403 consensus_queue.cc:1035] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:38.448135  2403 consensus_queue.cc:1035] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:38.453780   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.448982   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.458997  1229 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:38.462915  2412 tablet.cc:1722] T 58e7ec1f221f41ca8f6b019faeec3377 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:38.465545  1229 tablet_bootstrap.cc:654] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.473520  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:38.474164  1124 raft_consensus.cc:2466] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:38.488202   539 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_dea57d0c6e4149b1a063b0ec3b5bc013" } modify_external_catalogs: true
I20250628 07:52:38.488682   539 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_dea57d0c6e4149b1a063b0ec3b5bc013" } modify_external_catalogs: true
I20250628 07:52:38.492657   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ca0af8e0e69c4e2ba99ffa3917290323"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.494163   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ca0af8e0e69c4e2ba99ffa3917290323 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.477227   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.478803   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:38.497678   985 raft_consensus.cc:3058] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.498237  2403 consensus_queue.cc:1035] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:38.479486   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.479890   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.480438   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "547d0e51544c49fcbcfdcd05f2390025" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.480587   956 tablet_service.cc:1468] Processing CreateTablet for tablet 70584cf82896499ab43f7282337078ef (DEFAULT_TABLE table=default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 [id=839e2998d8e642f9a4fd0f7d376cefa4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.521034   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 70584cf82896499ab43f7282337078ef. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.522022   758 leader_election.cc:304] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
W20250628 07:52:38.522917  1062 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:38.523424  1062 leader_election.cc:343] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:38.523897  1062 leader_election.cc:343] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.481986   955 tablet_service.cc:1468] Processing CreateTablet for tablet d3e486f93a67441fa2da3a72af327503 (DEFAULT_TABLE table=default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.524927   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d3e486f93a67441fa2da3a72af327503. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.440232  2237 raft_consensus.cc:2241] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:38.528589  2237 raft_consensus.cc:2270] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.534714  2175 raft_consensus.cc:2802] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:38.535063  2175 raft_consensus.cc:491] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:38.535352  2175 raft_consensus.cc:3058] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.536935  2237 ts_tablet_manager.cc:1905] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.543817   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet e536268f648d4b0da1b9edc338ae974a
I20250628 07:52:38.551491  1234 ts_tablet_manager.cc:1428] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.360s	user 0.007s	sys 0.001s
I20250628 07:52:38.552651  1234 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:38.484227   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:38.505398   985 raft_consensus.cc:2466] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:38.512190   921 leader_election.cc:304] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:38.553148  2175 raft_consensus.cc:513] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.558851  2175 leader_election.cc:290] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:38.573216  2237 ts_tablet_manager.cc:1918] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:38.573581  2237 log.cc:1199] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:38.574215  2237 ts_tablet_manager.cc:1939] T fef70db0996f48c69574337f8f7ecf37 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:38.574563   804 tablet_service.cc:1515] Processing DeleteTablet for tablet e536268f648d4b0da1b9edc338ae974a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:38.574837  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 330) took 3701 ms. Trace:
W20250628 07:52:38.577605   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 6fdda1e4f7af468982a813f1ac023e01 consensus state has no leader
I20250628 07:52:38.578004  1230 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:38.578365  1230 ts_tablet_manager.cc:1397] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.451s	user 0.009s	sys 0.007s
I20250628 07:52:38.583673  1230 raft_consensus.cc:357] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.584376  1230 raft_consensus.cc:383] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.584631  1230 raft_consensus.cc:738] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.594977  1096 tablet_service.cc:1515] Processing DeleteTablet for tablet e536268f648d4b0da1b9edc338ae974a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:38.596943  1230 consensus_queue.cc:260] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.598491  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:38.613571  1122 raft_consensus.cc:3058] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.618182  2257 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 335) took 1043 ms. Trace:
I20250628 07:52:38.618487  2257 rpcz_store.cc:276] 0628 07:52:37.574689 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.602166 (+ 27477us) service_pool.cc:225] Handling call
0628 07:52:37.764042 (+161876us) ts_tablet_manager.cc:1145] Deleting tablet e15d063a4bca40d480cba08733ec1ec3
0628 07:52:37.764084 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:38.230867 (+466783us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.443643 (+212776us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.602279 (+158636us) spinlock_profiling.cc:243] Waited 82.6 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b4512 00007fbaa7d527e9 00007fbaa6fb2968 00007fbaa7077e62 00007fbaa70a7b70 00007fbaa708ef4d 00007fbaa708eef9 00007fbaa708ee89 00007fbaa708ee51
0628 07:52:38.618126 (+ 15847us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":14495,"spinlock_wait_cycles":189933952,"tablet-delete.queue_time_us":161342}
I20250628 07:52:38.620606  2257 tablet_replica.cc:331] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:38.621950   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_52441cbec305498ebed774e69004a383 [id=475e75d866ea4a5585227052d39db9da]) successfully deleted
W20250628 07:52:38.622496   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5fa4299a9e0f4b409d04c8c5900901c0 because the tablet was not found. No further retry: Not found: Tablet not found: 5fa4299a9e0f4b409d04c8c5900901c0
W20250628 07:52:38.622987   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fef70db0996f48c69574337f8f7ecf37 because the tablet was not found. No further retry: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37
W20250628 07:52:38.624682   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 6fdda1e4f7af468982a813f1ac023e01 (table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 6fdda1e4f7af468982a813f1ac023e01 consensus state has no leader
I20250628 07:52:38.625358  2257 raft_consensus.cc:2241] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:38.625339   957 tablet_service.cc:1468] Processing CreateTablet for tablet 1efec32bd8cd468eb13825d51722f7cc (DEFAULT_TABLE table=default.Table_786e13bb34124bc0bb4f5dae8326f725 [id=127fda336df94c48a181d1a59e0d2f00]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.625862  2257 raft_consensus.cc:2270] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.626354   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1efec32bd8cd468eb13825d51722f7cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.627399  2257 ts_tablet_manager.cc:1905] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.629547  1230 ts_tablet_manager.cc:1428] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.051s	user 0.005s	sys 0.000s
I20250628 07:52:38.630417  1230 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:38.636365  1122 raft_consensus.cc:2466] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:38.643204   965 tablet_service.cc:1468] Processing CreateTablet for tablet 9045c6953fb64629a2aeb59b498f6d9a (DEFAULT_TABLE table=default.Table_7631c6a0720642949d4b63e271a92e4c [id=46657c237cc041279dbb9e365d574c01]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.643204   963 tablet_service.cc:1468] Processing CreateTablet for tablet bf88267cb80545a5b4b2185732137576 (DEFAULT_TABLE table=default.Table_a90d3fd61b424016a14e3add816d2218 [id=57ab80d617474a998f26259768722a07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.643911  1234 tablet_bootstrap.cc:654] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.644193   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9045c6953fb64629a2aeb59b498f6d9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.644690   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bf88267cb80545a5b4b2185732137576. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.646080  2402 raft_consensus.cc:491] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.646476   958 tablet_service.cc:1515] Processing DeleteTablet for tablet e15d063a4bca40d480cba08733ec1ec3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.647557  2411 raft_consensus.cc:2747] T 547d0e51544c49fcbcfdcd05f2390025 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:38.646476  2402 raft_consensus.cc:513] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.654742   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8e3e0e52b4924e67904d3acfa4dd5752" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:38.655157   758 leader_election.cc:304] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:38.655328   825 raft_consensus.cc:2466] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:38.655814  2397 raft_consensus.cc:2802] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:38.656188  2397 raft_consensus.cc:695] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:38.656841  2397 consensus_queue.cc:237] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.669950  2411 raft_consensus.cc:491] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6a58514e01ea4989809bc2992fb2b806)
I20250628 07:52:38.660995  1230 tablet_bootstrap.cc:654] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.670117   920 leader_election.cc:304] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:38.670433  2411 raft_consensus.cc:513] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.671061  2337 raft_consensus.cc:2802] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:38.671376  2337 raft_consensus.cc:491] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:38.671700  2337 raft_consensus.cc:3058] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.671463   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_1382518892ee42e1a7f130402586ad3f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.673007   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_1382518892ee42e1a7f130402586ad3f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.674335  2411 leader_election.cc:290] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:38.627017   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 114979us
W20250628 07:52:38.675712   921 consensus_peers.cc:489] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: SHUTDOWN. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:38.680554  1176 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:38.575119  2237 rpcz_store.cc:276] 0628 07:52:34.873482 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:34.873600 (+   118us) service_pool.cc:225] Handling call
0628 07:52:38.439347 (+3565747us) ts_tablet_manager.cc:1145] Deleting tablet fef70db0996f48c69574337f8f7ecf37
0628 07:52:38.439381 (+    34us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:38.565637 (+126256us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.572995 (+  7358us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:38.574792 (+  1797us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3565206}
I20250628 07:52:38.689402   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_2dbb0bd965b34debae0cfb2c9f8db67b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.690920   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_2dbb0bd965b34debae0cfb2c9f8db67b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.692284  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 331) took 3686 ms. Trace:
I20250628 07:52:38.692675  2237 rpcz_store.cc:276] 0628 07:52:35.006258 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.023140 (+ 16882us) service_pool.cc:225] Handling call
0628 07:52:38.688148 (+3665008us) ts_tablet_manager.cc:1145] Deleting tablet d3501090b0a34ca9882dde55b19e3621
0628 07:52:38.691985 (+  3837us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:38.692218 (+   233us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3632531}
I20250628 07:52:38.693907  2237 tablet_replica.cc:331] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:38.694727  2237 raft_consensus.cc:2241] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:38.697775  1230 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:38.698145  1230 ts_tablet_manager.cc:1397] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.068s	user 0.008s	sys 0.000s
I20250628 07:52:38.700173  1230 raft_consensus.cc:357] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.701526  1230 raft_consensus.cc:383] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.701764  1230 raft_consensus.cc:738] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.704636  2257 ts_tablet_manager.cc:1918] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:38.704996  2257 log.cc:1199] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/63115b7be4a94b639c18601d438ed947
I20250628 07:52:38.705571  2257 ts_tablet_manager.cc:1939] T 63115b7be4a94b639c18601d438ed947 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:38.695195  2237 raft_consensus.cc:2270] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.705611  2395 consensus_queue.cc:1035] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.008s
I20250628 07:52:38.713972  1230 consensus_queue.cc:260] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.716069  1230 ts_tablet_manager.cc:1428] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.018s	user 0.004s	sys 0.000s
I20250628 07:52:38.717212   962 tablet_service.cc:1468] Processing CreateTablet for tablet 5f5d8a873a9f4197b9d5304ae08c12f3 (DEFAULT_TABLE table=default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf [id=222109d2472a4f079bc6e4abf9423ee6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.718127   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5d8a873a9f4197b9d5304ae08c12f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.722157   956 tablet_service.cc:1468] Processing CreateTablet for tablet d2578fbf2e974cc0bca65e32f45694ab (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.723042   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d2578fbf2e974cc0bca65e32f45694ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.723052   955 tablet_service.cc:1468] Processing CreateTablet for tablet 0bdb4c499be441379663830f26d80204 (DEFAULT_TABLE table=default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.724305   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0bdb4c499be441379663830f26d80204. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.726979   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 63115b7be4a94b639c18601d438ed947 (table default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]) successfully deleted
I20250628 07:52:38.728088  1230 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:38.730463   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 8f7dde9b060f4d839724e5f718519fb1 because the tablet was not found. No further retry: Not found: Tablet not found: 8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:38.731264  2257 tablet_replica.cc:331] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:38.733515  2257 raft_consensus.cc:2241] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:38.737401  2257 raft_consensus.cc:2270] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.739020  2257 ts_tablet_manager.cc:1905] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.743728   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_324d60f6b2914fdb85b95e4201ffee43"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.745188   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_324d60f6b2914fdb85b95e4201ffee43 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.736989  1230 tablet_bootstrap.cc:654] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.729578  2402 leader_election.cc:290] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:38.706425  2411 consensus_queue.cc:1035] T 95fa7c0a68d34d728042fb5a8a3b7307 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.044s
I20250628 07:52:38.779227   961 tablet_service.cc:1468] Processing CreateTablet for tablet c070c23e87594bf5853b2e42aafac7f5 (DEFAULT_TABLE table=default.Table_0b4dcd93c24b4479ad87780460b6afd8 [id=4fe25f3f36eb4a76addfc9f54bb85371]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.780289   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c070c23e87594bf5853b2e42aafac7f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.780988   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 3 got 2
I20250628 07:52:38.781904   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:38.782518   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:38.783105   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:38.783713   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:38.786029   546 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1382518892ee42e1a7f130402586ad3f" } modify_external_catalogs: true
I20250628 07:52:38.786506   546 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_1382518892ee42e1a7f130402586ad3f" } modify_external_catalogs: true
I20250628 07:52:38.790064  1229 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:38.790462  1229 ts_tablet_manager.cc:1397] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.332s	user 0.013s	sys 0.004s
I20250628 07:52:38.790267   545 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_2dbb0bd965b34debae0cfb2c9f8db67b" } new_table_name: "default.Table_8a1536aea0c64577a92168afbd496c5c" modify_external_catalogs: true
I20250628 07:52:38.793816   964 tablet_service.cc:1468] Processing CreateTablet for tablet e78cb8844a43404f9ee38dec2e532169 (DEFAULT_TABLE table=default.Table_b46689c124f54809961340f6c6b317d5 [id=bc2345b772a947e696627993d69e5e02]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.794813   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e78cb8844a43404f9ee38dec2e532169. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.794962  1230 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:38.795311  1230 ts_tablet_manager.cc:1397] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.067s	user 0.013s	sys 0.000s
I20250628 07:52:38.797502  1230 raft_consensus.cc:357] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.798156  1230 raft_consensus.cc:383] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.798396  1230 raft_consensus.cc:738] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.802779  2257 ts_tablet_manager.cc:1918] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:38.803150  2257 log.cc:1199] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:38.803730  2257 ts_tablet_manager.cc:1939] T 8155990eebd84fcf9ebc357e186d1bf2 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:38.806052   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 8155990eebd84fcf9ebc357e186d1bf2 (table default.Table_39092c471b2443d183b7229a1d625b66 [id=ab01e12b48e047828205036a96ce6762]) successfully deleted
I20250628 07:52:38.813331  1230 consensus_queue.cc:260] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.817838   952 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.818135   966 tablet_service.cc:1515] Processing DeleteTablet for tablet cc0bf0edebb140209531ccf917fd1ed8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.823124   959 tablet_service.cc:1515] Processing DeleteTablet for tablet 83ffa65248fc4db3aecc5f91b648b83f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.823201   953 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.825549   951 tablet_service.cc:1515] Processing DeleteTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 with delete_type TABLET_DATA_DELETED (replaced by tablet 80ee935a2d58422fb1bfbc9103940920) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.826444   954 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.829205   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 with delete_type TABLET_DATA_DELETED (replaced by tablet 586f1b7dd30e4998abebaa1fb2983e2b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:38.830717  1230 ts_tablet_manager.cc:1428] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.035s	user 0.005s	sys 0.000s
I20250628 07:52:38.831569  1230 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:38.832036   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 155122us
W20250628 07:52:38.836738  1062 leader_election.cc:343] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.837160  1062 leader_election.cc:304] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.837766  2162 raft_consensus.cc:2747] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:38.838119  1062 leader_election.cc:343] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.838455  1062 leader_election.cc:304] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.839023  2162 raft_consensus.cc:2747] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:38.839193   963 tablet_service.cc:1515] Processing DeleteTablet for tablet e536268f648d4b0da1b9edc338ae974a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:38.839298  1062 leader_election.cc:343] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.839622  1062 leader_election.cc:304] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.840170  2162 raft_consensus.cc:2747] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:38.840441  1062 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.840755  1062 leader_election.cc:304] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.841331  2162 raft_consensus.cc:2747] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:38.841650  1062 leader_election.cc:343] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.841987  1062 leader_election.cc:304] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.842530  2162 raft_consensus.cc:2747] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:38.843529  1062 leader_election.cc:343] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.843866  1062 leader_election.cc:304] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.844441  2162 raft_consensus.cc:2747] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:38.844596   546 catalog_manager.cc:5864] Not sending DeleteTablet requests; no consensus state for tablet dd837dcacd604c97a2acfda6491f921a
W20250628 07:52:38.846525   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because the tablet was not found. No further retry: Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
W20250628 07:52:38.847892   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet b51b51eb951d45a78bd2bffe47834b95 because the tablet was not found. No further retry: Not found: Tablet not found: b51b51eb951d45a78bd2bffe47834b95
I20250628 07:52:38.853601   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4f9faad6d70b4a559cfacf4e5deb27e3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.855080   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.855484  2428 tablet_replica.cc:331] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:38.855907   958 tablet_service.cc:1468] Processing CreateTablet for tablet aec37016d1674945b399cfc2f92b08fc (DEFAULT_TABLE table=default.Table_8244fa49d80542cca12bcda3b9225837 [id=4a58d7983dd146ae9e4ae45df899b0b6]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:38.856385  2428 raft_consensus.cc:2241] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:38.857066   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aec37016d1674945b399cfc2f92b08fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:38.859026  2257 tablet_replica.cc:331] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:38.859241   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 2ae890710f7f44cf899dd19e96fa9181 because the tablet was not found. No further retry: Not found: Tablet not found: 2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:38.859414  2337 raft_consensus.cc:513] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.859952  2257 raft_consensus.cc:2241] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:38.860327  2257 pending_rounds.cc:70] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Trying to abort 1 pending ops.
I20250628 07:52:38.860510  2257 pending_rounds.cc:77] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172493954014973952 op_type: NO_OP noop_request { }
I20250628 07:52:38.860796  2257 raft_consensus.cc:2887] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:38.861079  2257 raft_consensus.cc:2270] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.863473  2337 leader_election.cc:290] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:38.659442   757 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:38.867404  2257 ts_tablet_manager.cc:1905] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.784987   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:38.869321  2237 ts_tablet_manager.cc:1905] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.869395   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 58e7ec1f221f41ca8f6b019faeec3377 (table default.Table_59e9b18b772e4d5dafd78313d4b47800 [id=9a2c30b2d6a6409dbd3565a4b74ba210]). Expected version 1 got 0
I20250628 07:52:38.870419   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:38.871582   541 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:38.886430   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet fef70db0996f48c69574337f8f7ecf37 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]) successfully deleted
W20250628 07:52:38.887102   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet d3501090b0a34ca9882dde55b19e3621 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d3501090b0a34ca9882dde55b19e3621 already in progress: creating tablet
W20250628 07:52:38.887300   545 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 4d21243664ea4bbeb4a7c31d8c019486 has no consensus state
I20250628 07:52:38.892441   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fceb8ef4ddb44e7eb11be71a5087bd27" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:38.893009   983 raft_consensus.cc:3058] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:38.896835  1230 tablet_bootstrap.cc:654] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.899626  2428 raft_consensus.cc:2270] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:38.901242   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 67769.7us
I20250628 07:52:38.901913  2428 ts_tablet_manager.cc:1905] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.906481  1229 raft_consensus.cc:357] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.907110  1229 raft_consensus.cc:383] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.907356  1229 raft_consensus.cc:738] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.908000  1229 consensus_queue.cc:260] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:38.909255   921 leader_election.cc:343] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:38.911535   983 raft_consensus.cc:2466] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:38.916431  2257 ts_tablet_manager.cc:1918] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:38.916849  2257 log.cc:1199] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fa3ec14241e344029df3e39473181078
I20250628 07:52:38.917516  2257 ts_tablet_manager.cc:1939] T fa3ec14241e344029df3e39473181078 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:38.920254   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fa3ec14241e344029df3e39473181078 (table default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]) successfully deleted
I20250628 07:52:38.932283  1229 ts_tablet_manager.cc:1428] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.142s	user 0.003s	sys 0.001s
I20250628 07:52:38.933374  1229 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:38.934947  1230 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:38.935411  1230 ts_tablet_manager.cc:1397] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.104s	user 0.011s	sys 0.003s
W20250628 07:52:38.936527   921 leader_election.cc:343] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:38.937829  1230 raft_consensus.cc:357] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.943991   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3adf909f0f88430aa21e13480ae65522"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:38.945487  1230 raft_consensus.cc:383] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:38.945609   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3adf909f0f88430aa21e13480ae65522 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.945760  1230 raft_consensus.cc:738] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.945513  2330 consensus_queue.cc:1035] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:38.946918   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because the tablet was not found. No further retry: Not found: Tablet not found: cc0bf0edebb140209531ccf917fd1ed8
W20250628 07:52:38.948244   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fef70db0996f48c69574337f8f7ecf37 because the tablet was not found. No further retry: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37
W20250628 07:52:38.949518   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because the tablet was not found. No further retry: Not found: Tablet not found: 83ffa65248fc4db3aecc5f91b648b83f
W20250628 07:52:38.949663   921 leader_election.cc:343] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:38.950628  2410 raft_consensus.cc:491] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:38.951067  2410 raft_consensus.cc:513] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.953049  2410 leader_election.cc:290] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:38.954697   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:38.955561   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fef70db0996f48c69574337f8f7ecf37 because the tablet was not found. No further retry: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37
W20250628 07:52:38.956233  1062 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.950215  1230 consensus_queue.cc:260] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:38.958025   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because the tablet was not found. No further retry: Not found: Tablet not found: 46ff641001cd4b01a4c09c0d8b79ebf3
W20250628 07:52:38.958374   921 consensus_peers.cc:489] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fa3ec14241e344029df3e39473181078. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:38.960330  1230 ts_tablet_manager.cc:1428] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.025s	user 0.004s	sys 0.000s
I20250628 07:52:38.961750  1230 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:38.962452  1234 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:38.960902  2257 tablet_replica.cc:331] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:38.962860  1234 ts_tablet_manager.cc:1397] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.410s	user 0.006s	sys 0.009s
I20250628 07:52:38.963428  2257 raft_consensus.cc:2241] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:38.963943  2257 raft_consensus.cc:2270] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:38.968533  1234 raft_consensus.cc:357] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:38.965699  2257 ts_tablet_manager.cc:1905] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:38.969259  1234 raft_consensus.cc:383] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:38.969512  1234 raft_consensus.cc:738] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:38.971439   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_949d6d04ea9e4eb4a7af04bd4d201718"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:38.972930   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_949d6d04ea9e4eb4a7af04bd4d201718 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:38.975495  1230 tablet_bootstrap.cc:654] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:38.989738   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:38.991545  1062 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:38.991954  1062 leader_election.cc:304] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:38.992796  2410 raft_consensus.cc:2747] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:38.998926  1234 consensus_queue.cc:260] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.001312  1230 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.001689  1230 ts_tablet_manager.cc:1397] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.040s	user 0.006s	sys 0.003s
I20250628 07:52:39.003614  1234 ts_tablet_manager.cc:1428] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.041s	user 0.005s	sys 0.001s
I20250628 07:52:39.004438  1234 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:39.004438  1230 raft_consensus.cc:357] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.005023  1230 raft_consensus.cc:383] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.005299  1230 raft_consensus.cc:738] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.007921  2412 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:39.011719   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_d31add8b03f14004af81e01854d4ace9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.013252   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d31add8b03f14004af81e01854d4ace9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.013315  1230 consensus_queue.cc:260] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.015538  1230 ts_tablet_manager.cc:1428] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.014s	user 0.005s	sys 0.000s
I20250628 07:52:39.016304  1230 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.018788  2257 ts_tablet_manager.cc:1918] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:39.019169  2257 log.cc:1199] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:39.019868  2257 ts_tablet_manager.cc:1939] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:39.022086   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
W20250628 07:52:39.027139   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 1c65fa809b17418c85d4d72ae1e0ee3a because the tablet was not found. No further retry: Not found: Tablet not found: 1c65fa809b17418c85d4d72ae1e0ee3a
I20250628 07:52:39.033649  1230 tablet_bootstrap.cc:654] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.036768  2257 tablet_replica.cc:331] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:39.037745  2257 raft_consensus.cc:2241] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.038282  2257 raft_consensus.cc:2270] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.040261  2257 ts_tablet_manager.cc:1905] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:39.044384  1229 tablet_bootstrap.cc:654] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:39.066769   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 178662us
I20250628 07:52:39.066704  1230 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.074204  1230 ts_tablet_manager.cc:1397] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.058s	user 0.008s	sys 0.003s
I20250628 07:52:39.076530  1230 raft_consensus.cc:357] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.077251  1230 raft_consensus.cc:383] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.077500  1230 raft_consensus.cc:738] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.077572  2428 ts_tablet_manager.cc:1918] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20250628 07:52:39.078070  2428 log.cc:1199] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:39.078233  1230 consensus_queue.cc:260] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.078969  2428 ts_tablet_manager.cc:1939] T e15d063a4bca40d480cba08733ec1ec3 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:39.080885  1230 ts_tablet_manager.cc:1428] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20250628 07:52:39.081717  1230 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.083329  1229 tablet_bootstrap.cc:492] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:39.083710  1229 ts_tablet_manager.cc:1397] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.151s	user 0.008s	sys 0.000s
I20250628 07:52:39.086562  2444 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:39.096835   545 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_79306d05ef4945db94f6c1fbf588aa9d" } modify_external_catalogs: true
I20250628 07:52:39.097361   545 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_79306d05ef4945db94f6c1fbf588aa9d" } modify_external_catalogs: true
I20250628 07:52:39.099560  2410 raft_consensus.cc:491] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.100009  2410 raft_consensus.cc:513] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.101861  2410 leader_election.cc:290] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.103288   539 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_8244fa49d80542cca12bcda3b9225837" } new_table_name: "default.Table_bb80571048c94b099e096f7c712af383" modify_external_catalogs: true
I20250628 07:52:39.109539   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.110002   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_52441cbec305498ebed774e69004a383 [id=475e75d866ea4a5585227052d39db9da]) successfully deleted
I20250628 07:52:39.112278  1230 tablet_bootstrap.cc:654] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.112556  2412 tablet.cc:1722] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:39.113992  1062 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.119867  2428 tablet_replica.cc:331] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:39.120750  2428 raft_consensus.cc:2241] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.121176  1234 tablet_bootstrap.cc:654] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.121302  2428 raft_consensus.cc:2270] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.123525  2428 ts_tablet_manager.cc:1905] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:39.123283   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.124660   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.125447   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.130890   539 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet aec37016d1674945b399cfc2f92b08fc consensus state has no leader
W20250628 07:52:39.133307  1062 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.133955  1062 leader_election.cc:304] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.134625  2410 raft_consensus.cc:2747] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.135530  2440 raft_consensus.cc:491] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.136039  2440 raft_consensus.cc:513] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.138444  1230 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.138597  2440 leader_election.cc:290] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.138859  1230 ts_tablet_manager.cc:1397] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.057s	user 0.006s	sys 0.004s
W20250628 07:52:39.139724   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 62107.4us
I20250628 07:52:39.141045  1230 raft_consensus.cc:357] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.141739  1230 raft_consensus.cc:383] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.142043  1230 raft_consensus.cc:738] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.142709  1230 consensus_queue.cc:260] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.144102  2439 raft_consensus.cc:491] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 82f43cdfab544b17a6a71033983e65d4)
I20250628 07:52:39.144538  2439 raft_consensus.cc:513] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.146476  2439 leader_election.cc:290] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.150080  2448 tablet.cc:1722] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:39.158994  1230 ts_tablet_manager.cc:1428] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.020s	user 0.005s	sys 0.000s
I20250628 07:52:39.159935  1230 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.170071   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8e3e0e52b4924e67904d3acfa4dd5752" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:39.179229   920 leader_election.cc:343] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:39.179677   920 leader_election.cc:304] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:39.188287   757 leader_election.cc:304] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.193603   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_49b746aa19f841728b4cf05fa450c090"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.194914   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_49b746aa19f841728b4cf05fa450c090 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.196738   539 master_service.cc:948] ReplaceTablet: received request to replace tablet de7ff3260b0049d682dba82f914474de from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:39.197588  2392 raft_consensus.cc:2747] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20250628 07:52:39.202234   921 consensus_peers.cc:489] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:39.190668  2237 ts_tablet_manager.cc:1918] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
W20250628 07:52:39.207233   920 leader_election.cc:343] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:52:39.207530   920 leader_election.cc:304] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:39.205899  2237 log.cc:1199] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/8e3e0e52b4924e67904d3acfa4dd5752
W20250628 07:52:39.195971   758 leader_election.cc:343] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:39.211725  2403 raft_consensus.cc:2747] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.212273  1230 tablet_bootstrap.cc:654] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:39.215489  2369 lock_manager.cc:585] Waited 2 seconds to obtain row lock on key '<redacted>' (key hash 15122197062527616531) tablet 00000000000000000000000000000000 cur holder 0x7b5800054600
I20250628 07:52:39.219573  2237 ts_tablet_manager.cc:1939] T 8e3e0e52b4924e67904d3acfa4dd5752 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:39.223714   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet de7ff3260b0049d682dba82f914474de
I20250628 07:52:39.225441  2410 consensus_queue.cc:1035] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:39.228390  2330 raft_consensus.cc:2747] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.230870   955 tablet_service.cc:1515] Processing DeleteTablet for tablet de7ff3260b0049d682dba82f914474de with delete_type TABLET_DATA_DELETED (replaced by tablet b7a634cdcdbb46aa9d3f79b834ea7d37) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:39.231117   539 catalog_manager.cc:6652] ReplaceTablet: tablet de7ff3260b0049d682dba82f914474de deleted and replaced by tablet b7a634cdcdbb46aa9d3f79b834ea7d37
I20250628 07:52:39.231909  1096 tablet_service.cc:1515] Processing DeleteTablet for tablet de7ff3260b0049d682dba82f914474de with delete_type TABLET_DATA_DELETED (replaced by tablet b7a634cdcdbb46aa9d3f79b834ea7d37) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:39.236191   758 consensus_peers.cc:489] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9ba2a10acdf0462bb05628351cdbbbe4. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:39.237136   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_168a40d10a3645bdaea410044e99f357"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.238900   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_168a40d10a3645bdaea410044e99f357 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.242226  2257 ts_tablet_manager.cc:1918] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:39.242717  2257 log.cc:1199] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/8f7e8d9640c74c8da5ac00adaf86f8a1
I20250628 07:52:39.243492  2257 ts_tablet_manager.cc:1939] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:39.246955  1230 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.247340  1230 ts_tablet_manager.cc:1397] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.088s	user 0.005s	sys 0.007s
I20250628 07:52:39.249775  1230 raft_consensus.cc:357] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.250376  1230 raft_consensus.cc:383] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.250648  1230 raft_consensus.cc:738] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.250653   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]) successfully deleted
I20250628 07:52:39.251207  1230 consensus_queue.cc:260] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.251901   804 tablet_service.cc:1515] Processing DeleteTablet for tablet de7ff3260b0049d682dba82f914474de with delete_type TABLET_DATA_DELETED (replaced by tablet b7a634cdcdbb46aa9d3f79b834ea7d37) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:39.261207  2410 raft_consensus.cc:491] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.261612  2410 raft_consensus.cc:513] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.263443  2410 leader_election.cc:290] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.265317   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.266638  1062 leader_election.cc:343] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.277181   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e393a3a8856046f1a8d11d7ab8c485ee"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.278661   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e393a3a8856046f1a8d11d7ab8c485ee in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.287770  2410 raft_consensus.cc:491] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.288257  2410 raft_consensus.cc:513] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:39.288625   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet e536268f648d4b0da1b9edc338ae974a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e536268f648d4b0da1b9edc338ae974a already in progress: creating tablet
I20250628 07:52:39.290099  2410 leader_election.cc:290] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.295395   873 heartbeater.cc:344] Connected to a master server at 127.0.107.189:33023
I20250628 07:52:39.309392  2257 tablet_replica.cc:331] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:39.310350  2257 raft_consensus.cc:2241] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.317451   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8155990eebd84fcf9ebc357e186d1bf2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.317644  1229 raft_consensus.cc:357] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.318225  1229 raft_consensus.cc:383] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.310961  2257 raft_consensus.cc:2270] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.318475  1229 raft_consensus.cc:738] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.319072  1229 consensus_queue.cc:260] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.320040   985 raft_consensus.cc:2466] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:39.323381  2410 raft_consensus.cc:491] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.323807  2410 raft_consensus.cc:513] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:39.325953   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 72310.2us
I20250628 07:52:39.326072   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.331486  1229 ts_tablet_manager.cc:1428] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.247s	user 0.007s	sys 0.000s
I20250628 07:52:39.332382  1229 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:39.333995   546 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_d81430dc148d442e8dbfc9029c7cc3c1" } new_table_name: "default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1" modify_external_catalogs: true
I20250628 07:52:39.337126  2410 leader_election.cc:290] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.342773   539 master_service.cc:948] ReplaceTablet: received request to replace tablet d2578fbf2e974cc0bca65e32f45694ab from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:39.352254   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.353488  1062 leader_election.cc:343] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.360335  1230 ts_tablet_manager.cc:1428] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.113s	user 0.004s	sys 0.000s
I20250628 07:52:39.362843   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.363960  1062 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.364722  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 332) took 4160 ms. Trace:
I20250628 07:52:39.365170  2237 rpcz_store.cc:276] 0628 07:52:35.204515 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.206087 (+  1572us) service_pool.cc:225] Handling call
0628 07:52:38.693750 (+3487663us) ts_tablet_manager.cc:1145] Deleting tablet 8e3e0e52b4924e67904d3acfa4dd5752
0628 07:52:38.693793 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:38.906182 (+212389us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.190407 (+284225us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.364668 (+174261us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":161060,"tablet-delete.queue_time_us":3487057}
I20250628 07:52:39.365703  1230 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.368126   545 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 35fe2e8156cc43d7b3c02092877063b6
I20250628 07:52:39.372179  1096 tablet_service.cc:1515] Processing DeleteTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:39 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:39.372458   545 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_9287e3b4d3614c488fb35212433f337f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:39.376708   545 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9287e3b4d3614c488fb35212433f337f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:39.377655  2411 raft_consensus.cc:491] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.378110  2411 raft_consensus.cc:513] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.379889  2411 leader_election.cc:290] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.380473  1230 tablet_bootstrap.cc:654] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.381279  2444 tablet.cc:1722] T 58e7ec1f221f41ca8f6b019faeec3377 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:39.383623   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 61055.4us
I20250628 07:52:39.384269  2237 tablet_replica.cc:331] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:39.385464  2237 raft_consensus.cc:2241] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.386433   757 leader_election.cc:304] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:39.387264  2330 raft_consensus.cc:2802] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:39.387578  2330 raft_consensus.cc:491] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:39.387761  2257 ts_tablet_manager.cc:1905] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:39.387899  2330 raft_consensus.cc:3058] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:39.389252  1062 leader_election.cc:343] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.389636  1062 leader_election.cc:304] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.390833  2457 raft_consensus.cc:2747] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.392311   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 8e3e0e52b4924e67904d3acfa4dd5752 (table default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]) successfully deleted
I20250628 07:52:39.394074   804 tablet_service.cc:1515] Processing DeleteTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:39 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:39.397768  2330 raft_consensus.cc:513] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.399472  2330 leader_election.cc:290] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:39.369582   758 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8f7dde9b060f4d839724e5f718519fb1
W20250628 07:52:39.400967   757 leader_election.cc:343] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: 8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:39.401662   757 leader_election.cc:304] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.402925  2440 raft_consensus.cc:2704] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Received pre-election callback for term 2 while not running: Illegal state: RaftConsensus is not running: State = Stopping
I20250628 07:52:39.395340   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:39.409315   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.411101   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8155990eebd84fcf9ebc357e186d1bf2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:39.411573   985 raft_consensus.cc:3058] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:39.419804  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:39.420369  1129 raft_consensus.cc:2466] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:39.422394   921 leader_election.cc:304] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:39.423897  2457 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.424371  2457 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:39.424928  1062 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.425303  1062 leader_election.cc:304] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.426390  2457 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.427953  2337 raft_consensus.cc:2802] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:39.428272  2337 raft_consensus.cc:491] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:39.428565  2337 raft_consensus.cc:3058] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:39.430603   539 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet d2578fbf2e974cc0bca65e32f45694ab
I20250628 07:52:39.435712   955 tablet_service.cc:1515] Processing DeleteTablet for tablet d2578fbf2e974cc0bca65e32f45694ab with delete_type TABLET_DATA_DELETED (replaced by tablet 06b9b63649384313948964b3c0dd11cf) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:39.436275  1096 tablet_service.cc:1515] Processing DeleteTablet for tablet d2578fbf2e974cc0bca65e32f45694ab with delete_type TABLET_DATA_DELETED (replaced by tablet 06b9b63649384313948964b3c0dd11cf) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:39.437247  2330 consensus_queue.cc:1035] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:39.431142   958 tablet_service.cc:1515] Processing DeleteTablet for tablet 35fe2e8156cc43d7b3c02092877063b6 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:39 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:39.438650   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.441139   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.442775   539 catalog_manager.cc:6652] ReplaceTablet: tablet d2578fbf2e974cc0bca65e32f45694ab deleted and replaced by tablet 06b9b63649384313948964b3c0dd11cf
I20250628 07:52:39.445047  2237 raft_consensus.cc:2270] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.446225  1230 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.446601  1230 ts_tablet_manager.cc:1397] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.083s	user 0.005s	sys 0.004s
I20250628 07:52:39.447090  2237 ts_tablet_manager.cc:1905] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:39.452042   758 consensus_peers.cc:489] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 63115b7be4a94b639c18601d438ed947. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:39.448709  1230 raft_consensus.cc:357] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.453563  1230 raft_consensus.cc:383] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.453822  1230 raft_consensus.cc:738] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:39.454051   920 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.454380  1230 consensus_queue.cc:260] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.456398  1230 ts_tablet_manager.cc:1428] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.010s	user 0.003s	sys 0.000s
I20250628 07:52:39.458760   804 tablet_service.cc:1515] Processing DeleteTablet for tablet d2578fbf2e974cc0bca65e32f45694ab with delete_type TABLET_DATA_DELETED (replaced by tablet 06b9b63649384313948964b3c0dd11cf) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:39.464267  1230 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.372292  2459 tablet.cc:1722] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:39.466193   958 tablet_service.cc:1468] Processing CreateTablet for tablet d38887520de2440d8155a19a8954f9cd (DEFAULT_TABLE table=default.Table_9ac8a7257de34accbdf87e0548648d25 [id=a70a1a3bd354432e9ebf6211b0a7a5b7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.467283  1234 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:39.467425   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d38887520de2440d8155a19a8954f9cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.467684  1234 ts_tablet_manager.cc:1397] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.463s	user 0.016s	sys 0.013s
I20250628 07:52:39.470405  1234 raft_consensus.cc:357] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.471047  1234 raft_consensus.cc:383] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.471280  1234 raft_consensus.cc:738] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.471798  1234 consensus_queue.cc:260] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.475947   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 3 got 2
I20250628 07:52:39.478606  1234 ts_tablet_manager.cc:1428] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.011s	user 0.002s	sys 0.005s
I20250628 07:52:39.479007  1096 tablet_service.cc:1468] Processing CreateTablet for tablet d38887520de2440d8155a19a8954f9cd (DEFAULT_TABLE table=default.Table_9ac8a7257de34accbdf87e0548648d25 [id=a70a1a3bd354432e9ebf6211b0a7a5b7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.480072  1096 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d38887520de2440d8155a19a8954f9cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.480422  1234 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:39.486058  1234 tablet_bootstrap.cc:654] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.487581  1229 tablet_bootstrap.cc:654] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.426421  2410 raft_consensus.cc:2747] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.497903  2462 tablet.cc:1722] T 60179e6765b145adbf637a8435eef724 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:39.500995  1230 tablet_bootstrap.cc:654] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.504072  1097 tablet_service.cc:1468] Processing CreateTablet for tablet c426c5a91c7e43658b158fd16b9fa60c (DEFAULT_TABLE table=default.Table_ea4026116e0e482e84f4f48211e0079c [id=b91ffe1c2ac54b0f81f812bf9c13a641]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.505347   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:39.513835  1230 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.514256  1230 ts_tablet_manager.cc:1397] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.050s	user 0.009s	sys 0.001s
I20250628 07:52:39.516301  2457 raft_consensus.cc:491] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.495322   985 raft_consensus.cc:2466] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:39.516801  2457 raft_consensus.cc:513] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.517885   757 leader_election.cc:304] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:39.518615  2330 raft_consensus.cc:2802] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:39.519039  2330 raft_consensus.cc:695] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:39.518823  2457 leader_election.cc:290] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:39.519807  1230 raft_consensus.cc:357] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.519853  2330 consensus_queue.cc:237] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.520622  1230 raft_consensus.cc:383] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.521932  1230 raft_consensus.cc:738] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.523898  1234 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:39.524325  1234 ts_tablet_manager.cc:1397] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.044s	user 0.010s	sys 0.001s
I20250628 07:52:39.497901  2337 raft_consensus.cc:513] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.502743   955 tablet_service.cc:1468] Processing CreateTablet for tablet c426c5a91c7e43658b158fd16b9fa60c (DEFAULT_TABLE table=default.Table_ea4026116e0e482e84f4f48211e0079c [id=b91ffe1c2ac54b0f81f812bf9c13a641]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.526117   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c426c5a91c7e43658b158fd16b9fa60c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.526511  1234 raft_consensus.cc:357] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.527078  1234 raft_consensus.cc:383] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.527304  1234 raft_consensus.cc:738] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.521409  2457 consensus_queue.cc:1035] T 547d0e51544c49fcbcfdcd05f2390025 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:39.521023  1062 leader_election.cc:343] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.528679  1062 leader_election.cc:304] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:39.529626  1062 leader_election.cc:343] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:39.531986   758 leader_election.cc:343] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:39.533288   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.534967  1062 leader_election.cc:343] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.535306  1062 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.535934  2410 raft_consensus.cc:2747] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:39.536193  1062 leader_election.cc:343] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.536646  2410 raft_consensus.cc:2747] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.538066   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:39.537973  1230 consensus_queue.cc:260] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:39.539252  1062 leader_election.cc:343] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.539613  1062 leader_election.cc:304] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.538821  1234 consensus_queue.cc:260] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.542290  1234 ts_tablet_manager.cc:1428] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.018s	user 0.004s	sys 0.000s
I20250628 07:52:39.543120  1234 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:39.543991   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:39.544188   804 tablet_service.cc:1468] Processing CreateTablet for tablet d38887520de2440d8155a19a8954f9cd (DEFAULT_TABLE table=default.Table_9ac8a7257de34accbdf87e0548648d25 [id=a70a1a3bd354432e9ebf6211b0a7a5b7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.545168   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d38887520de2440d8155a19a8954f9cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.545327  2410 raft_consensus.cc:2747] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.550796  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:39.551301  1124 raft_consensus.cc:3058] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:39.543112  1230 ts_tablet_manager.cc:1428] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.001s
I20250628 07:52:39.552743  1230 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.553346  2337 leader_election.cc:290] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.554605  1094 tablet_service.cc:1468] Processing CreateTablet for tablet 8bcc01e64a874cd69300e3b78e2d9ab8 (DEFAULT_TABLE table=default.Table_c22cd1bf6d884c33bf39513fb5da717a [id=2f1a522e39a24e749aaaa0578ad3c6c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.558308   962 tablet_service.cc:1468] Processing CreateTablet for tablet 8bcc01e64a874cd69300e3b78e2d9ab8 (DEFAULT_TABLE table=default.Table_c22cd1bf6d884c33bf39513fb5da717a [id=2f1a522e39a24e749aaaa0578ad3c6c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.559379   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bcc01e64a874cd69300e3b78e2d9ab8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.476822   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:39.563221   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:39.564366   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:39.564697  2410 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.565392  1124 raft_consensus.cc:2466] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:39.565618   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:39.567037   921 leader_election.cc:304] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:39.565172  2410 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.572924  1234 tablet_bootstrap.cc:654] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.575516  2337 raft_consensus.cc:2802] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:39.578553  2410 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:39.607987   794 tablet_service.cc:1468] Processing CreateTablet for tablet c426c5a91c7e43658b158fd16b9fa60c (DEFAULT_TABLE table=default.Table_ea4026116e0e482e84f4f48211e0079c [id=b91ffe1c2ac54b0f81f812bf9c13a641]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.610357   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c426c5a91c7e43658b158fd16b9fa60c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.611974  1097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c426c5a91c7e43658b158fd16b9fa60c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.613495  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bcc01e64a874cd69300e3b78e2d9ab8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.621369   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:39.622958  1062 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.575629   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:39.627130   920 leader_election.cc:343] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:39.629863  1062 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.630196  1062 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.631091  2410 raft_consensus.cc:2747] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.638825  2237 ts_tablet_manager.cc:1918] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:39.639169  2237 log.cc:1199] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:39.632227  2337 raft_consensus.cc:695] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:39.653719  2337 consensus_queue.cc:237] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.654237   962 tablet_service.cc:1468] Processing CreateTablet for tablet 71c5bdface3d44ecae33bef018e7f18d (DEFAULT_TABLE table=default.Table_848608eca1b140e5bb004562ebd3fde8 [id=700083322b4a4b118855057493210fb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.655129   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c5bdface3d44ecae33bef018e7f18d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.655653  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 71c5bdface3d44ecae33bef018e7f18d (DEFAULT_TABLE table=default.Table_848608eca1b140e5bb004562ebd3fde8 [id=700083322b4a4b118855057493210fb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.657994  1229 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:39.567006   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:39.658344  1229 ts_tablet_manager.cc:1397] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.326s	user 0.009s	sys 0.004s
I20250628 07:52:39.659415  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c5bdface3d44ecae33bef018e7f18d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.666426   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:39.667086   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 58e7ec1f221f41ca8f6b019faeec3377 (table default.Table_59e9b18b772e4d5dafd78313d4b47800 [id=9a2c30b2d6a6409dbd3565a4b74ba210]). Expected version 1 got 0
I20250628 07:52:39.668118   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:39.675241   805 tablet_service.cc:1468] Processing CreateTablet for tablet 7e3210014d264071bfc2960a7406bcd7 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.676101   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e3210014d264071bfc2960a7406bcd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.679173  2440 consensus_queue.cc:1035] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:39.695776  2257 ts_tablet_manager.cc:1918] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:39.696168  2257 log.cc:1199] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/de7ff3260b0049d682dba82f914474de
I20250628 07:52:39.697427  2257 ts_tablet_manager.cc:1939] T de7ff3260b0049d682dba82f914474de P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:39.660389  1229 raft_consensus.cc:357] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.700692  1229 raft_consensus.cc:383] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.701110  1229 raft_consensus.cc:738] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.705401  1230 tablet_bootstrap.cc:654] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.707325   795 tablet_service.cc:1468] Processing CreateTablet for tablet 71c5bdface3d44ecae33bef018e7f18d (DEFAULT_TABLE table=default.Table_848608eca1b140e5bb004562ebd3fde8 [id=700083322b4a4b118855057493210fb7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.708248   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71c5bdface3d44ecae33bef018e7f18d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.701881  1229 consensus_queue.cc:260] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.714867  1234 tablet_bootstrap.cc:492] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:39.715154  1229 ts_tablet_manager.cc:1428] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.057s	user 0.006s	sys 0.000s
I20250628 07:52:39.715206  1234 ts_tablet_manager.cc:1397] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.172s	user 0.012s	sys 0.002s
I20250628 07:52:39.715929  1229 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:39.730021  1230 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:39.730391  1230 ts_tablet_manager.cc:1397] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.178s	user 0.012s	sys 0.000s
I20250628 07:52:39.738757  2428 ts_tablet_manager.cc:1918] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:39.743517  1230 raft_consensus.cc:357] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.744238  1230 raft_consensus.cc:383] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.739212  2428 log.cc:1199] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:39.746698  2410 raft_consensus.cc:491] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.747031  2428 ts_tablet_manager.cc:1939] T 46ff641001cd4b01a4c09c0d8b79ebf3 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:39.747146  2410 raft_consensus.cc:513] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.748940  2410 leader_election.cc:290] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:39.751118  1229 tablet_bootstrap.cc:654] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:39.751650   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:39.752871  1062 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.761274  2428 tablet_replica.cc:331] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:39.762154  2428 raft_consensus.cc:2241] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.762682  2428 raft_consensus.cc:2270] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.764703   962 tablet_service.cc:1468] Processing CreateTablet for tablet 0d0c851c1efb4774a834b0662b092f6c (DEFAULT_TABLE table=default.Table_d3fc569c0e854f788cd8355607639d3a [id=12c2d9bf9bc8422a9f415382c8202daa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.765961   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0d0c851c1efb4774a834b0662b092f6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.768200   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 46ff641001cd4b01a4c09c0d8b79ebf3 (table default.Table_402049f057084c25a8b2a78d7fbdc2a0 [id=979223400058407baeaba004d05ab8ad]) successfully deleted
W20250628 07:52:39.769176   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because the tablet was not found. No further retry: Not found: Tablet not found: cc0bf0edebb140209531ccf917fd1ed8
W20250628 07:52:39.769938   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because the tablet was not found. No further retry: Not found: Tablet not found: 83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:39.770253   955 tablet_service.cc:1468] Processing CreateTablet for tablet 460693e542d7452c9f80777dbff67b57 (DEFAULT_TABLE table=default.Table_e04e4751022f4e2fbc686f49b598150e [id=14de477c143d4d2084393ef5d658e4d0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.771364   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 460693e542d7452c9f80777dbff67b57. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:39.772229   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 6fdda1e4f7af468982a813f1ac023e01 (table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 6fdda1e4f7af468982a813f1ac023e01 consensus state has no leader
W20250628 07:52:39.772229   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet aec37016d1674945b399cfc2f92b08fc (table default.Table_bb80571048c94b099e096f7c712af383 [id=4a58d7983dd146ae9e4ae45df899b0b6], current schema version=4) failed: Not found: failed to reset TS proxy: No leader found: Tablet aec37016d1674945b399cfc2f92b08fc consensus state has no leader
I20250628 07:52:39.776226   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet de7ff3260b0049d682dba82f914474de (table default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]) successfully deleted
I20250628 07:52:39.778440  1094 tablet_service.cc:1468] Processing CreateTablet for tablet 7e3210014d264071bfc2960a7406bcd7 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.779211  1096 tablet_service.cc:1468] Processing CreateTablet for tablet 0d0c851c1efb4774a834b0662b092f6c (DEFAULT_TABLE table=default.Table_d3fc569c0e854f788cd8355607639d3a [id=12c2d9bf9bc8422a9f415382c8202daa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.779525  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e3210014d264071bfc2960a7406bcd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.780210  1096 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0d0c851c1efb4774a834b0662b092f6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.780994  1097 tablet_service.cc:1468] Processing CreateTablet for tablet 460693e542d7452c9f80777dbff67b57 (DEFAULT_TABLE table=default.Table_e04e4751022f4e2fbc686f49b598150e [id=14de477c143d4d2084393ef5d658e4d0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.781996  1097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 460693e542d7452c9f80777dbff67b57. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.785225  2440 consensus_queue.cc:1035] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:39.785769   958 tablet_service.cc:1468] Processing CreateTablet for tablet 7e3210014d264071bfc2960a7406bcd7 (DEFAULT_TABLE table=default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.787068   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e3210014d264071bfc2960a7406bcd7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.788200   797 tablet_service.cc:1468] Processing CreateTablet for tablet 0d0c851c1efb4774a834b0662b092f6c (DEFAULT_TABLE table=default.Table_d3fc569c0e854f788cd8355607639d3a [id=12c2d9bf9bc8422a9f415382c8202daa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.788985  2411 raft_consensus.cc:491] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:39.789247   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0d0c851c1efb4774a834b0662b092f6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.789422  2411 raft_consensus.cc:513] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.791254  2411 leader_election.cc:290] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.792694  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:39.793293  1129 raft_consensus.cc:2466] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:39.794298   921 leader_election.cc:304] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:39.794993  2411 raft_consensus.cc:2802] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:39.795275  2411 raft_consensus.cc:491] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:39.795562  2411 raft_consensus.cc:3058] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:39.797499   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 35fe2e8156cc43d7b3c02092877063b6 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 35fe2e8156cc43d7b3c02092877063b6 already in progress: creating tablet
W20250628 07:52:39.798496   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d2578fbf2e974cc0bca65e32f45694ab because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d2578fbf2e974cc0bca65e32f45694ab already in progress: creating tablet
I20250628 07:52:39.801313   804 tablet_service.cc:1468] Processing CreateTablet for tablet 8bcc01e64a874cd69300e3b78e2d9ab8 (DEFAULT_TABLE table=default.Table_c22cd1bf6d884c33bf39513fb5da717a [id=2f1a522e39a24e749aaaa0578ad3c6c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.802333   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8bcc01e64a874cd69300e3b78e2d9ab8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.804332  2411 raft_consensus.cc:513] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.744488  1230 raft_consensus.cc:738] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.806257  1230 consensus_queue.cc:260] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.788249  1234 raft_consensus.cc:357] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.810402  1234 raft_consensus.cc:383] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.810757  1234 raft_consensus.cc:738] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.811369  1234 consensus_queue.cc:260] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:39.815121   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 73717.6us
W20250628 07:52:39.815899   758 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 52284.4us
I20250628 07:52:39.819669   798 tablet_service.cc:1468] Processing CreateTablet for tablet 460693e542d7452c9f80777dbff67b57 (DEFAULT_TABLE table=default.Table_e04e4751022f4e2fbc686f49b598150e [id=14de477c143d4d2084393ef5d658e4d0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:39.820895   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:39.821908   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:39.822746   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:39.823974   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:39.824569   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
W20250628 07:52:39.825443  1062 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:39.825845  1062 leader_election.cc:304] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:39.827780  2410 raft_consensus.cc:2747] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:39.842440  1230 ts_tablet_manager.cc:1428] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.112s	user 0.005s	sys 0.002s
I20250628 07:52:39.843426  1230 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:39.853403   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:39.867393  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:39.867947  1126 raft_consensus.cc:3058] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:39.868651  2411 leader_election.cc:290] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:39.875635   547 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:39.880910  1229 tablet_bootstrap.cc:492] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:39.881343  1229 ts_tablet_manager.cc:1397] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.166s	user 0.009s	sys 0.001s
I20250628 07:52:39.883306  1229 raft_consensus.cc:357] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.883860  1229 raft_consensus.cc:383] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:39.884056  1229 raft_consensus.cc:738] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:39.639835  2237 ts_tablet_manager.cc:1939] T 8f7dde9b060f4d839724e5f718519fb1 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:39.911077  2428 ts_tablet_manager.cc:1905] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:39.915310  1234 ts_tablet_manager.cc:1428] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.200s	user 0.006s	sys 0.000s
I20250628 07:52:39.915606   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 460693e542d7452c9f80777dbff67b57. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:39.919135   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:39.919725   817 raft_consensus.cc:2466] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:39.920560   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "86591ef37c8547189ff8ef270e9bfa9a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:39.921037   824 raft_consensus.cc:3058] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:39.924737   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 49961us
I20250628 07:52:39.933856  1234 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
W20250628 07:52:39.936965   757 consensus_peers.cc:489] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:39.940428  1126 raft_consensus.cc:2466] T 86591ef37c8547189ff8ef270e9bfa9a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:39.943331   921 leader_election.cc:304] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:39.943478   824 raft_consensus.cc:2466] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:39.944115  2337 raft_consensus.cc:2802] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:39.944677  2337 raft_consensus.cc:695] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:39.945614  2337 consensus_queue.cc:237] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:39.947999  2428 ts_tablet_manager.cc:1918] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:39.952422  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 333) took 4093 ms. Trace:
I20250628 07:52:39.948357  2428 log.cc:1199] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:39.964138  2428 ts_tablet_manager.cc:1939] T 9ba2a10acdf0462bb05628351cdbbbe4 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:39.966543  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 379) took 1171 ms. Trace:
I20250628 07:52:39.966984  2428 rpcz_store.cc:276] 0628 07:52:38.795125 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.823558 (+ 28433us) service_pool.cc:225] Handling call
0628 07:52:39.761125 (+937567us) ts_tablet_manager.cc:1145] Deleting tablet 9ba2a10acdf0462bb05628351cdbbbe4
0628 07:52:39.761160 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:39.930111 (+168951us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.947768 (+ 17657us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.966467 (+ 18699us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":33868,"tablet-delete.queue_time_us":935115}
I20250628 07:52:39.952855  2237 rpcz_store.cc:276] 0628 07:52:35.858584 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:35.859004 (+   420us) service_pool.cc:225] Handling call
0628 07:52:39.384128 (+3525124us) ts_tablet_manager.cc:1145] Deleting tablet 8f7dde9b060f4d839724e5f718519fb1
0628 07:52:39.384164 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:39.571284 (+187120us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.638632 (+ 67348us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:39.952360 (+313728us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3524528}
I20250628 07:52:39.969906  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 335) took 3895 ms. Trace:
I20250628 07:52:39.970211  2237 rpcz_store.cc:276] 0628 07:52:36.074200 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.074341 (+   141us) service_pool.cc:225] Handling call
0628 07:52:39.968786 (+3894445us) ts_tablet_manager.cc:1145] Deleting tablet 711843da7df74ba2bb22d386bc477a38
0628 07:52:39.969612 (+   826us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:39.969852 (+   240us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3893928}
I20250628 07:52:39.974123  2428 tablet_replica.cc:331] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:39.974965  2428 raft_consensus.cc:2241] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:39.975450  2428 raft_consensus.cc:2270] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:39.980588  2428 ts_tablet_manager.cc:1905] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:39.991240  2237 tablet_replica.cc:331] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:39.996701  1229 consensus_queue.cc:260] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.002094  1230 tablet_bootstrap.cc:654] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:40.003388   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 46734.8us
I20250628 07:52:40.003890  1033 heartbeater.cc:344] Connected to a master server at 127.0.107.190:37209
I20250628 07:52:40.011405  2237 raft_consensus.cc:2241] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:40.012328  1229 ts_tablet_manager.cc:1428] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.131s	user 0.006s	sys 0.000s
I20250628 07:52:40.013531  1229 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:40.018345  1234 tablet_bootstrap.cc:654] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.019215  2483 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:40.020579  2237 raft_consensus.cc:2270] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:40.024026  2237 ts_tablet_manager.cc:1905] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:40.035737  2410 raft_consensus.cc:491] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.050941  2410 raft_consensus.cc:513] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.059391  2410 leader_election.cc:290] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:40.064640   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:40.067718  1062 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.068369   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:40.069567  1062 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.070204  1062 leader_election.cc:304] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:40.076143  2410 raft_consensus.cc:2747] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:40.078279  2459 tablet.cc:1722] T 79d15e600b8b460fab54eb21193f1f40 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:40.084672  2483 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
W20250628 07:52:40.030997   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.CreateTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 75778.3us
I20250628 07:52:40.096179   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
I20250628 07:52:40.099165   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 8f7dde9b060f4d839724e5f718519fb1 (table default.Table_bb80571048c94b099e096f7c712af383 [id=4a58d7983dd146ae9e4ae45df899b0b6]) successfully deleted
W20250628 07:52:40.110678   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 711843da7df74ba2bb22d386bc477a38 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 711843da7df74ba2bb22d386bc477a38 already in progress: creating tablet
I20250628 07:52:40.117604  2410 raft_consensus.cc:491] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.118058  2410 raft_consensus.cc:513] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.120380  2483 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:40.122421  2142 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 499 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:40.126963  1230 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:40.127390  1230 ts_tablet_manager.cc:1397] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.284s	user 0.004s	sys 0.006s
I20250628 07:52:40.127858   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:40.128899  1062 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.129484  2410 leader_election.cc:290] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:40.130753   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 499 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:40.132030   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:40.133309  1062 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.133692  1062 leader_election.cc:304] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:40.134325  2410 raft_consensus.cc:2747] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:40.136569   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:40.146093  1230 raft_consensus.cc:357] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.146929  1230 raft_consensus.cc:383] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.139078  2483 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:40.147218  1230 raft_consensus.cc:738] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:40.157948  1230 consensus_queue.cc:260] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.166853  2428 ts_tablet_manager.cc:1918] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:40.167248  2428 log.cc:1199] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:40.167913  2428 ts_tablet_manager.cc:1939] T ee03996105074308bda56d36ba3a1c5d P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:40.169344  2494 tablet.cc:1722] T 79d15e600b8b460fab54eb21193f1f40 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:40.134331  2495 tablet.cc:1722] T 79d15e600b8b460fab54eb21193f1f40 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:40.174860  1229 tablet_bootstrap.cc:654] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.168615  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 376) took 1374 ms. Trace:
I20250628 07:52:40.177343  2428 rpcz_store.cc:276] 0628 07:52:38.794160 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.823498 (+ 29338us) service_pool.cc:225] Handling call
0628 07:52:39.973537 (+1150039us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:39.973989 (+   452us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.058957 (+ 84968us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.166636 (+107679us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.168564 (+  1928us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":30498,"tablet-delete.queue_time_us":1146665}
I20250628 07:52:40.178511   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet ee03996105074308bda56d36ba3a1c5d (table default.Table_85406b18e2c04407b80adfc0e5ee9587 [id=d2e8fc95ce2040f3a5adb94ba4f7c61d]) successfully deleted
I20250628 07:52:40.174032  1230 ts_tablet_manager.cc:1428] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.046s	user 0.007s	sys 0.001s
I20250628 07:52:40.199983  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 377) took 1405 ms. Trace:
I20250628 07:52:40.177253  1234 tablet_bootstrap.cc:492] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:40.200350  2428 rpcz_store.cc:276] 0628 07:52:38.794459 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.818391 (+ 23932us) service_pool.cc:225] Handling call
0628 07:52:40.199703 (+1381312us) ts_tablet_manager.cc:1145] Deleting tablet fef70db0996f48c69574337f8f7ecf37
0628 07:52:40.199754 (+    51us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.199923 (+   169us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":30501,"tablet-delete.queue_time_us":1372762}
I20250628 07:52:40.201239  1234 ts_tablet_manager.cc:1397] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.268s	user 0.011s	sys 0.003s
I20250628 07:52:40.203868  1234 raft_consensus.cc:357] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:40.217877   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet fef70db0996f48c69574337f8f7ecf37 because the tablet was not found. No further retry: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:40.218899  2428 tablet_replica.cc:331] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:40.219837  2428 raft_consensus.cc:2241] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:40.220736  2428 raft_consensus.cc:2270] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:40.234179  2402 raft_consensus.cc:491] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.234601  2402 raft_consensus.cc:513] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.237246  2402 leader_election.cc:290] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.258272  2428 ts_tablet_manager.cc:1905] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:40.270979   958 tablet_service.cc:1468] Processing CreateTablet for tablet c3d1bc8483374a2a917451a43b3d42ed (DEFAULT_TABLE table=default.Table_9287e3b4d3614c488fb35212433f337f [id=c13ed2379df1474e9b067d848f08e9b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.272135   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3d1bc8483374a2a917451a43b3d42ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.273921  1096 tablet_service.cc:1468] Processing CreateTablet for tablet c3d1bc8483374a2a917451a43b3d42ed (DEFAULT_TABLE table=default.Table_9287e3b4d3614c488fb35212433f337f [id=c13ed2379df1474e9b067d848f08e9b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.275269   962 tablet_service.cc:1468] Processing CreateTablet for tablet 21ee5fbf572c4530939c1c103b89737e (DEFAULT_TABLE table=default.Table_e393a3a8856046f1a8d11d7ab8c485ee [id=3311e522b275451fa31744d77fcc8654]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.276144   962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 21ee5fbf572c4530939c1c103b89737e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.276213  1229 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:40.276614  1229 ts_tablet_manager.cc:1397] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.263s	user 0.009s	sys 0.000s
I20250628 07:52:40.278046  2402 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.280452  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:40.281775  1124 raft_consensus.cc:2466] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:40.285050   921 leader_election.cc:304] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:40.285993  2337 raft_consensus.cc:2802] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:40.286294  2337 raft_consensus.cc:491] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:40.286595  2337 raft_consensus.cc:3058] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.279507  2402 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.291078   955 tablet_service.cc:1468] Processing CreateTablet for tablet f49a0de38fe24baab09728a19513878f (DEFAULT_TABLE table=default.Table_49b746aa19f841728b4cf05fa450c090 [id=a12bdfa9daa947988fdabac0b95e453a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.292013   955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f49a0de38fe24baab09728a19513878f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.297075  1094 tablet_service.cc:1468] Processing CreateTablet for tablet f49a0de38fe24baab09728a19513878f (DEFAULT_TABLE table=default.Table_49b746aa19f841728b4cf05fa450c090 [id=a12bdfa9daa947988fdabac0b95e453a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.298424  1100 tablet_service.cc:1468] Processing CreateTablet for tablet 21ee5fbf572c4530939c1c103b89737e (DEFAULT_TABLE table=default.Table_e393a3a8856046f1a8d11d7ab8c485ee [id=3311e522b275451fa31744d77fcc8654]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.299640  2402 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.299232   545 catalog_manager.cc:5582] T 547d0e51544c49fcbcfdcd05f2390025 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "6a58514e01ea4989809bc2992fb2b806" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
W20250628 07:52:40.300673   921 consensus_peers.cc:489] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: de7ff3260b0049d682dba82f914474de. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:40.302520  1097 tablet_service.cc:1468] Processing CreateTablet for tablet 2c932ba2738a414da4405677d886dabe (DEFAULT_TABLE table=default.Table_d31add8b03f14004af81e01854d4ace9 [id=79aa7a6bdf0f407f97ad9fc126ba6e8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.306762  1095 tablet_service.cc:1468] Processing CreateTablet for tablet 33ae7e653d644224a07a3bf29ef9fcd9 (DEFAULT_TABLE table=default.Table_949d6d04ea9e4eb4a7af04bd4d201718 [id=637c356534054cef9cdada8dfa2cf907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.308534   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b]). Expected version 3 got 2
I20250628 07:52:40.311990  1099 tablet_service.cc:1468] Processing CreateTablet for tablet ec358ab4881d42ad8ffbcf3464ef59a3 (DEFAULT_TABLE table=default.Table_3adf909f0f88430aa21e13480ae65522 [id=75edd41d3e954616b79662475b416941]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.227427  2488 raft_consensus.cc:491] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.312942   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:40.313344  2488 raft_consensus.cc:513] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.314370   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:40.314860   964 tablet_service.cc:1468] Processing CreateTablet for tablet 2c932ba2738a414da4405677d886dabe (DEFAULT_TABLE table=default.Table_d31add8b03f14004af81e01854d4ace9 [id=79aa7a6bdf0f407f97ad9fc126ba6e8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.315155  2488 leader_election.cc:290] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.238581   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:40.315758   964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c932ba2738a414da4405677d886dabe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.316066   825 raft_consensus.cc:2466] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:40.316599   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:40.317157   956 tablet_service.cc:1468] Processing CreateTablet for tablet 33ae7e653d644224a07a3bf29ef9fcd9 (DEFAULT_TABLE table=default.Table_949d6d04ea9e4eb4a7af04bd4d201718 [id=637c356534054cef9cdada8dfa2cf907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.318013   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33ae7e653d644224a07a3bf29ef9fcd9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.319052  1105 tablet_service.cc:1468] Processing CreateTablet for tablet a4900b15c92a4024917480a3fce8373c (DEFAULT_TABLE table=default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 [id=54d01002870a488baf5a808adbfbc006]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.320412   805 tablet_service.cc:1468] Processing CreateTablet for tablet 2c932ba2738a414da4405677d886dabe (DEFAULT_TABLE table=default.Table_d31add8b03f14004af81e01854d4ace9 [id=79aa7a6bdf0f407f97ad9fc126ba6e8c]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.321156   796 tablet_service.cc:1468] Processing CreateTablet for tablet ec358ab4881d42ad8ffbcf3464ef59a3 (DEFAULT_TABLE table=default.Table_3adf909f0f88430aa21e13480ae65522 [id=75edd41d3e954616b79662475b416941]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.321396   797 tablet_service.cc:1468] Processing CreateTablet for tablet 33ae7e653d644224a07a3bf29ef9fcd9 (DEFAULT_TABLE table=default.Table_949d6d04ea9e4eb4a7af04bd4d201718 [id=637c356534054cef9cdada8dfa2cf907]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.280109   795 tablet_service.cc:1468] Processing CreateTablet for tablet 21ee5fbf572c4530939c1c103b89737e (DEFAULT_TABLE table=default.Table_e393a3a8856046f1a8d11d7ab8c485ee [id=3311e522b275451fa31744d77fcc8654]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.321656   957 tablet_service.cc:1468] Processing CreateTablet for tablet ec358ab4881d42ad8ffbcf3464ef59a3 (DEFAULT_TABLE table=default.Table_3adf909f0f88430aa21e13480ae65522 [id=75edd41d3e954616b79662475b416941]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.320353   794 tablet_service.cc:1468] Processing CreateTablet for tablet f49a0de38fe24baab09728a19513878f (DEFAULT_TABLE table=default.Table_49b746aa19f841728b4cf05fa450c090 [id=a12bdfa9daa947988fdabac0b95e453a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.322213   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]). Expected version 1 got 0
I20250628 07:52:40.322757   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec358ab4881d42ad8ffbcf3464ef59a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.323607   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:40.273744   798 tablet_service.cc:1468] Processing CreateTablet for tablet c3d1bc8483374a2a917451a43b3d42ed (DEFAULT_TABLE table=default.Table_9287e3b4d3614c488fb35212433f337f [id=c13ed2379df1474e9b067d848f08e9b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.324630  2337 raft_consensus.cc:513] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.325659   963 tablet_service.cc:1468] Processing CreateTablet for tablet a4900b15c92a4024917480a3fce8373c (DEFAULT_TABLE table=default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 [id=54d01002870a488baf5a808adbfbc006]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.326500   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4900b15c92a4024917480a3fce8373c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.326936   803 tablet_service.cc:1468] Processing CreateTablet for tablet b1da78a71a634d408fb5de4efe92ebbe (DEFAULT_TABLE table=default.Table_324d60f6b2914fdb85b95e4201ffee43 [id=0b6ea0c021b3450ba01083f4373e29e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.328090   800 tablet_service.cc:1468] Processing CreateTablet for tablet a4900b15c92a4024917480a3fce8373c (DEFAULT_TABLE table=default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 [id=54d01002870a488baf5a808adbfbc006]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.328047  1098 tablet_service.cc:1468] Processing CreateTablet for tablet b1da78a71a634d408fb5de4efe92ebbe (DEFAULT_TABLE table=default.Table_324d60f6b2914fdb85b95e4201ffee43 [id=0b6ea0c021b3450ba01083f4373e29e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.329999  2496 raft_consensus.cc:491] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.330642  2496 raft_consensus.cc:513] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.331746   953 tablet_service.cc:1468] Processing CreateTablet for tablet 6ce66199d1614826bf618819f6928383 (DEFAULT_TABLE table=default.Table_ca0af8e0e69c4e2ba99ffa3917290323 [id=7fd88f265a2942be8ce28347d572c18d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.331986   966 tablet_service.cc:1468] Processing CreateTablet for tablet b1da78a71a634d408fb5de4efe92ebbe (DEFAULT_TABLE table=default.Table_324d60f6b2914fdb85b95e4201ffee43 [id=0b6ea0c021b3450ba01083f4373e29e8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.332937   953 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ce66199d1614826bf618819f6928383. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.332937   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:40.333456   825 raft_consensus.cc:2466] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:40.334319   920 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:40.335484  2402 raft_consensus.cc:2802] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:52:40.335825  2402 raft_consensus.cc:491] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:40.336707  2402 raft_consensus.cc:3058] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Advancing to term 2
W20250628 07:52:40.346813   921 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:40.342861  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:40.347385   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1da78a71a634d408fb5de4efe92ebbe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.325040   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:40.352052   793 tablet_service.cc:1468] Processing CreateTablet for tablet 6ce66199d1614826bf618819f6928383 (DEFAULT_TABLE table=default.Table_ca0af8e0e69c4e2ba99ffa3917290323 [id=7fd88f265a2942be8ce28347d572c18d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.337220  2496 leader_election.cc:290] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:40.360042  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 6ce66199d1614826bf618819f6928383 (DEFAULT_TABLE table=default.Table_ca0af8e0e69c4e2ba99ffa3917290323 [id=7fd88f265a2942be8ce28347d572c18d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.363380   545 catalog_manager.cc:5582] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:40.366809   801 tablet_service.cc:1468] Processing CreateTablet for tablet f8f42a3a559b448f828d724383043a12 (DEFAULT_TABLE table=default.Table_77748d06f2a54596a43e8a89b5b56011 [id=a32a0dbf0f3a44ff8703768610951a4a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.371901  1093 tablet_service.cc:1468] Processing CreateTablet for tablet f8f42a3a559b448f828d724383043a12 (DEFAULT_TABLE table=default.Table_77748d06f2a54596a43e8a89b5b56011 [id=a32a0dbf0f3a44ff8703768610951a4a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.373420   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:40.373948   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c932ba2738a414da4405677d886dabe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.375583  1230 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:40.377588   796 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec358ab4881d42ad8ffbcf3464ef59a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.381563   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33ae7e653d644224a07a3bf29ef9fcd9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.385622   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 21ee5fbf572c4530939c1c103b89737e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.387501   802 tablet_service.cc:1468] Processing CreateTablet for tablet cc49fd5696e1403e95ea8d60220eafa0 (DEFAULT_TABLE table=default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 [id=beca5a78fa254e6eb8732613e47278fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.388496   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc49fd5696e1403e95ea8d60220eafa0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.389269   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f49a0de38fe24baab09728a19513878f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.278903  1229 raft_consensus.cc:357] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.407187  1229 raft_consensus.cc:383] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.407449  1229 raft_consensus.cc:738] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:40.408012  1229 consensus_queue.cc:260] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.409822   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:40.410084   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:40.410890   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1da78a71a634d408fb5de4efe92ebbe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.411370   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4900b15c92a4024917480a3fce8373c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.411890   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f42a3a559b448f828d724383043a12. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.412935   793 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ce66199d1614826bf618819f6928383. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.415356  1229 ts_tablet_manager.cc:1428] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.139s	user 0.001s	sys 0.008s
I20250628 07:52:40.422305   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:40.422834   824 raft_consensus.cc:3058] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.421895  2337 leader_election.cc:290] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:40.423815   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 77031.4us
I20250628 07:52:40.425365   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3d1bc8483374a2a917451a43b3d42ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.428550   799 tablet_service.cc:1468] Processing CreateTablet for tablet b1fcc569fb1b4a84983b651074e3e64d (DEFAULT_TABLE table=default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 [id=69980641f09443da87a0d85cb5cca3aa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.429518   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1fcc569fb1b4a84983b651074e3e64d. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:40.436002   757 leader_election.cc:343] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.441715   804 tablet_service.cc:1468] Processing CreateTablet for tablet 06b9b63649384313948964b3c0dd11cf (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.442740   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06b9b63649384313948964b3c0dd11cf. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:40.443956  1062 leader_election.cc:343] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.444093  1102 tablet_service.cc:1468] Processing CreateTablet for tablet cc49fd5696e1403e95ea8d60220eafa0 (DEFAULT_TABLE table=default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 [id=beca5a78fa254e6eb8732613e47278fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.447641   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:40.448766  1062 leader_election.cc:343] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.450058  1092 tablet_service.cc:1468] Processing CreateTablet for tablet b1fcc569fb1b4a84983b651074e3e64d (DEFAULT_TABLE table=default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 [id=69980641f09443da87a0d85cb5cca3aa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.451892  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cacde13bc29458da81f29fb4fa2945a" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:40.452358  1127 raft_consensus.cc:3058] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.458343  1230 tablet_bootstrap.cc:654] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.458750  1124 raft_consensus.cc:2466] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:40.459122   824 raft_consensus.cc:2466] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:40.449169  1062 leader_election.cc:304] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:40.469519   920 leader_election.cc:304] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:40.470800  2496 raft_consensus.cc:2747] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:40.471089  2411 raft_consensus.cc:2802] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:40.471438   758 leader_election.cc:304] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:40.471516  2411 raft_consensus.cc:695] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:40.472247  2411 consensus_queue.cc:237] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.472704  2439 raft_consensus.cc:2802] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:40.468966  1127 raft_consensus.cc:2466] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:40.473023  2439 raft_consensus.cc:491] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:40.473354  2439 raft_consensus.cc:3058] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.480983  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 06b9b63649384313948964b3c0dd11cf (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.483357  2439 raft_consensus.cc:513] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.487182  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 64f5ca605ce4480da9eaad08456ddb9a (DEFAULT_TABLE table=default.Table_d86e7bc180df48c0b8358485d2a8bc2f [id=693a9f55cd4d4fcca72d0d27c3cc8b07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.491160   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:40.495210   961 tablet_service.cc:1468] Processing CreateTablet for tablet cc49fd5696e1403e95ea8d60220eafa0 (DEFAULT_TABLE table=default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 [id=beca5a78fa254e6eb8732613e47278fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.495210   954 tablet_service.cc:1468] Processing CreateTablet for tablet f8f42a3a559b448f828d724383043a12 (DEFAULT_TABLE table=default.Table_77748d06f2a54596a43e8a89b5b56011 [id=a32a0dbf0f3a44ff8703768610951a4a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.495385   792 tablet_service.cc:1468] Processing CreateTablet for tablet 3cfb8eefaa45432bb50cc729a401a2ec (DEFAULT_TABLE table=default.Table_4ee15e08b0884908a81ad107f6738fe8 [id=cce2fcb4460e4fef8025a77e355c1890]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.496892   951 tablet_service.cc:1468] Processing CreateTablet for tablet b1fcc569fb1b4a84983b651074e3e64d (DEFAULT_TABLE table=default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 [id=69980641f09443da87a0d85cb5cca3aa]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.497058   960 tablet_service.cc:1468] Processing CreateTablet for tablet 06b9b63649384313948964b3c0dd11cf (DEFAULT_TABLE table=default.Table_00d339485c8d4aa38db91013d26fdcc5 [id=039bc5279ecd4e90bf5938f626b4efdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.498415   959 tablet_service.cc:1468] Processing CreateTablet for tablet 64f5ca605ce4480da9eaad08456ddb9a (DEFAULT_TABLE table=default.Table_d86e7bc180df48c0b8358485d2a8bc2f [id=693a9f55cd4d4fcca72d0d27c3cc8b07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.498492   965 tablet_service.cc:1468] Processing CreateTablet for tablet 3cfb8eefaa45432bb50cc729a401a2ec (DEFAULT_TABLE table=default.Table_4ee15e08b0884908a81ad107f6738fe8 [id=cce2fcb4460e4fef8025a77e355c1890]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.499267  2432 raft_consensus.cc:491] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.499656  2432 raft_consensus.cc:513] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.501488  2432 leader_election.cc:290] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.502923  1229 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:40.504043   960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06b9b63649384313948964b3c0dd11cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.503998  1104 tablet_service.cc:1468] Processing CreateTablet for tablet 3cfb8eefaa45432bb50cc729a401a2ec (DEFAULT_TABLE table=default.Table_4ee15e08b0884908a81ad107f6738fe8 [id=cce2fcb4460e4fef8025a77e355c1890]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.505198   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:40.510129   920 leader_election.cc:343] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.511440   792 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3cfb8eefaa45432bb50cc729a401a2ec. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.517709   959 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 64f5ca605ce4480da9eaad08456ddb9a. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:40.518107   920 consensus_peers.cc:489] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:40.518344   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3cfb8eefaa45432bb50cc729a401a2ec. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.521582   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc49fd5696e1403e95ea8d60220eafa0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.525518   954 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f42a3a559b448f828d724383043a12. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.529430   951 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1fcc569fb1b4a84983b651074e3e64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.531778  1234 raft_consensus.cc:383] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.532045  1234 raft_consensus.cc:738] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:40.532673  1234 consensus_queue.cc:260] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.534009   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:40.534593  1234 ts_tablet_manager.cc:1428] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.333s	user 0.004s	sys 0.000s
I20250628 07:52:40.535811  2439 leader_election.cc:290] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.536880  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:40.537458  1127 raft_consensus.cc:3058] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.541827   803 tablet_service.cc:1468] Processing CreateTablet for tablet 64f5ca605ce4480da9eaad08456ddb9a (DEFAULT_TABLE table=default.Table_d86e7bc180df48c0b8358485d2a8bc2f [id=693a9f55cd4d4fcca72d0d27c3cc8b07]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.536456   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:40.544044   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:40.545603  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:40.546195  1124 raft_consensus.cc:2466] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:40.550618   758 consensus_peers.cc:489] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8155990eebd84fcf9ebc357e186d1bf2. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:40.558847   985 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f845f32699044867af6f0f2b655cad8e" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:40.566129  2441 consensus_queue.cc:1035] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:40.569845   921 leader_election.cc:304] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:40.571467  2392 raft_consensus.cc:2802] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:40.571796  2392 raft_consensus.cc:491] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:40.572089  2392 raft_consensus.cc:3058] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.579432  2392 raft_consensus.cc:513] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.582618   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:40.582729  2392 leader_election.cc:290] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.583959   791 tablet_service.cc:1468] Processing CreateTablet for tablet 4d21243664ea4bbeb4a7c31d8c019486 (DEFAULT_TABLE table=default.Table_8a1536aea0c64577a92168afbd496c5c [id=5bbb2a3c166044139f30f8650e698936]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:40.584846   920 leader_election.cc:343] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.589298  2402 raft_consensus.cc:513] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.596814  1109 tablet_service.cc:1468] Processing CreateTablet for tablet 4d21243664ea4bbeb4a7c31d8c019486 (DEFAULT_TABLE table=default.Table_8a1536aea0c64577a92168afbd496c5c [id=5bbb2a3c166044139f30f8650e698936]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.598143   790 tablet_service.cc:1468] Processing CreateTablet for tablet 80ee935a2d58422fb1bfbc9103940920 (DEFAULT_TABLE table=default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.600442  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:40.600951  1126 raft_consensus.cc:3058] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:40.602303  1107 tablet_service.cc:1468] Processing CreateTablet for tablet 80ee935a2d58422fb1bfbc9103940920 (DEFAULT_TABLE table=default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.603744  1229 tablet_bootstrap.cc:654] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.605099   789 tablet_service.cc:1468] Processing CreateTablet for tablet b7a634cdcdbb46aa9d3f79b834ea7d37 (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.607105   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2ae890710f7f44cf899dd19e96fa9181" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:40.607522  1126 raft_consensus.cc:2466] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:40.607656   824 raft_consensus.cc:3058] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:52:40.616129  1127 raft_consensus.cc:2466] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:40.617033   758 leader_election.cc:304] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:40.617708  1928 raft_consensus.cc:2802] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:40.618055  1928 raft_consensus.cc:695] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:40.618461   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:40.618657  1928 consensus_queue.cc:237] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.619997   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:40.620769   539 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]). Expected version 1 got 0
I20250628 07:52:40.620887  2402 leader_election.cc:290] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.622401  2495 tablet.cc:1722] T 8feee3be1a664cae9da417b190163a94 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:40.629005  1101 tablet_service.cc:1468] Processing CreateTablet for tablet b7a634cdcdbb46aa9d3f79b834ea7d37 (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.631217  2510 tablet.cc:1722] T 8feee3be1a664cae9da417b190163a94 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
I20250628 07:52:40.645618   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet d87dd96d00964e12acd6e380a62c94d8
I20250628 07:52:40.647233   543 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 404) took 2942 ms. Trace:
I20250628 07:52:40.649267  2328 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:40.649875  2328 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:40.645030  2428 ts_tablet_manager.cc:1918] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:40.651731  2428 log.cc:1199] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/8f7e8d9640c74c8da5ac00adaf86f8a1
I20250628 07:52:40.651818  1091 tablet_service.cc:1515] Processing DeleteTablet for tablet d87dd96d00964e12acd6e380a62c94d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:40.652480  2428 ts_tablet_manager.cc:1939] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:40.653638   802 tablet_service.cc:1515] Processing DeleteTablet for tablet d87dd96d00964e12acd6e380a62c94d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.647562   543 rpcz_store.cc:276] 0628 07:52:37.704892 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.139801 (+434909us) service_pool.cc:225] Handling call
0628 07:52:38.271811 (+132010us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:38.431158 (+159347us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:38.431274 (+   116us) catalog_manager.cc:2799] Locking tablets
0628 07:52:38.434108 (+  2834us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:38.501304 (+ 67196us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.501328 (+    24us) write_op.cc:454] Released schema lock
0628 07:52:38.501549 (+   221us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:38.501715 (+   166us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:40.645375 (+2143660us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:40.647129 (+  1754us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:38.436497 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:38.436870 (+   373us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:38.436890 (+    20us) write_op.cc:435] Acquired schema lock
0628 07:52:38.436900 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:38.437181 (+   281us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:38.437216 (+    35us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:38.437234 (+    18us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:38.437453 (+   219us) tablet.cc:699] Row locks acquired
0628 07:52:38.437464 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:38.437554 (+    90us) write_op.cc:270] Start()
0628 07:52:38.437626 (+    72us) write_op.cc:276] Timestamp: P: 1751097158437498 usec, L: 0
0628 07:52:38.437640 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:38.438085 (+   445us) log.cc:844] Serialized 796 byte log entry
0628 07:52:38.449989 (+ 11904us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.450292 (+   303us) write_op.cc:301] APPLY: starting
0628 07:52:38.450347 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.450437 (+    90us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.450548 (+   111us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.450812 (+   264us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.450967 (+   155us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:38.451011 (+    44us) write_op.cc:312] APPLY: finished
0628 07:52:38.451178 (+   167us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.451324 (+   146us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.451387 (+    63us) write_op.cc:454] Released schema lock
0628 07:52:38.451416 (+    29us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":167,"apply.run_cpu_time_us":1321,"apply.run_wall_time_us":1321,"num_ops":2,"prepare.queue_time_us":476,"prepare.run_cpu_time_us":1827,"prepare.run_wall_time_us":1827,"raft.queue_time_us":2943,"raft.run_cpu_time_us":491,"raft.run_wall_time_us":491,"replication_time_us":12270}]]}
I20250628 07:52:40.655056  2368 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 3 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:40.655845  2368 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:40.665119  2328 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:40.665647  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 380) took 1843 ms. Trace:
I20250628 07:52:40.666069  2428 rpcz_store.cc:276] 0628 07:52:38.822530 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.822636 (+   106us) service_pool.cc:225] Handling call
0628 07:52:40.218728 (+1396092us) ts_tablet_manager.cc:1145] Deleting tablet 8f7e8d9640c74c8da5ac00adaf86f8a1
0628 07:52:40.218774 (+    46us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.573281 (+354507us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.644811 (+ 71530us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.665581 (+ 20770us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":27973,"tablet-delete.queue_time_us":1389154}
I20250628 07:52:40.669417  1229 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:40.670224  1229 ts_tablet_manager.cc:1397] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.168s	user 0.008s	sys 0.002s
I20250628 07:52:40.670904   544 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 403) took 2982 ms. Trace:
I20250628 07:52:40.672546  1229 raft_consensus.cc:357] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.673179  1229 raft_consensus.cc:383] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.673424  1229 raft_consensus.cc:738] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:40.676319  2439 raft_consensus.cc:491] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.676764  2439 raft_consensus.cc:513] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.678900  2439 leader_election.cc:290] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:40.684574  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:40.685400  1127 raft_consensus.cc:2391] T 0b251095244548629ae93d52e6c80ccc P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:40.685652  1229 consensus_queue.cc:260] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.688269  1229 ts_tablet_manager.cc:1428] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.018s	user 0.005s	sys 0.000s
I20250628 07:52:40.689471  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 381) took 1855 ms. Trace:
I20250628 07:52:40.689807  2428 rpcz_store.cc:276] 0628 07:52:38.833751 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.839066 (+  5315us) service_pool.cc:225] Handling call
0628 07:52:40.688920 (+1849854us) ts_tablet_manager.cc:1145] Deleting tablet e536268f648d4b0da1b9edc338ae974a
0628 07:52:40.688962 (+    42us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.689417 (+   455us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":20543,"tablet-delete.queue_time_us":1849371}
I20250628 07:52:40.690889  2428 tablet_replica.cc:331] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:40.691730  2428 raft_consensus.cc:2241] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:40.692605  2428 raft_consensus.cc:2270] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:40.694612  2428 ts_tablet_manager.cc:1905] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:40.696655  1229 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:40.696831   757 leader_election.cc:343] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.699447   921 leader_election.cc:304] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:40.700177  2411 raft_consensus.cc:2802] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:40.700255   802 tablet_service.cc:1515] Processing DeleteTablet for tablet fef70db0996f48c69574337f8f7ecf37 with delete_type TABLET_DATA_DELETED (replaced by tablet d3e486f93a67441fa2da3a72af327503) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.700608  2411 raft_consensus.cc:695] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:40.701484  2411 consensus_queue.cc:237] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.701833  1090 tablet_service.cc:1468] Processing CreateTablet for tablet ca128104712e4b98b6bea2cb4bd604f2 (DEFAULT_TABLE table=default.Table_168a40d10a3645bdaea410044e99f357 [id=2013562180c34579a816dde8278e9c85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.703864   802 tablet_service.cc:1468] Processing CreateTablet for tablet 586f1b7dd30e4998abebaa1fb2983e2b (DEFAULT_TABLE table=default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.704582  1229 tablet_bootstrap.cc:654] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.705061   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 586f1b7dd30e4998abebaa1fb2983e2b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.671201   544 rpcz_store.cc:276] 0628 07:52:37.688669 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.017143 (+328474us) service_pool.cc:225] Handling call
0628 07:52:38.018249 (+  1106us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:38.110643 (+ 92394us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:38.245158 (+134515us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:38.245405 (+   247us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:38.311195 (+ 65790us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:40.670713 (+2359518us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:38.246460 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:38.246838 (+   378us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:38.246860 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:38.246871 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:38.247080 (+   209us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:38.247114 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:38.247134 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:38.247281 (+   147us) tablet.cc:699] Row locks acquired
0628 07:52:38.247294 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:38.247390 (+    96us) write_op.cc:270] Start()
0628 07:52:38.247459 (+    69us) write_op.cc:276] Timestamp: P: 1751097158247351 usec, L: 0
0628 07:52:38.247474 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:38.263064 (+ 15590us) log.cc:844] Serialized 405 byte log entry
0628 07:52:38.264663 (+  1599us) spinlock_profiling.cc:243] Waited 15 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:38.306899 (+ 42236us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.307297 (+   398us) write_op.cc:301] APPLY: starting
0628 07:52:38.307346 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.307424 (+    78us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.307440 (+    16us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.307579 (+   139us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.307699 (+   120us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:38.307736 (+    37us) write_op.cc:312] APPLY: finished
0628 07:52:38.307898 (+   162us) log.cc:844] Serialized 23 byte log entry
0628 07:52:38.308471 (+   573us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.308528 (+    57us) write_op.cc:454] Released schema lock
0628 07:52:38.308555 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:38.308745 (+   190us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.308775 (+    30us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":282,"apply.run_cpu_time_us":982,"apply.run_wall_time_us":1416,"num_ops":1,"prepare.queue_time_us":85,"prepare.run_cpu_time_us":3464,"prepare.run_wall_time_us":18305,"replication_time_us":59351,"spinlock_wait_cycles":34573696}]]}
I20250628 07:52:40.709364   795 tablet_service.cc:1515] Processing DeleteTablet for tablet 8f7dde9b060f4d839724e5f718519fb1 with delete_type TABLET_DATA_DELETED (replaced by tablet aec37016d1674945b399cfc2f92b08fc) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.714779  1230 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:40.715248  1230 ts_tablet_manager.cc:1397] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.340s	user 0.007s	sys 0.003s
I20250628 07:52:40.728818   795 tablet_service.cc:1515] Processing DeleteTablet for tablet 1c65fa809b17418c85d4d72ae1e0ee3a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.730063  1234 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:40.730221   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 64f5ca605ce4480da9eaad08456ddb9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.732442   951 tablet_service.cc:1468] Processing CreateTablet for tablet 80ee935a2d58422fb1bfbc9103940920 (DEFAULT_TABLE table=default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.733667   805 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.734860   951 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80ee935a2d58422fb1bfbc9103940920. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.735445   797 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.745852   791 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d21243664ea4bbeb4a7c31d8c019486. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.749640   790 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80ee935a2d58422fb1bfbc9103940920. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.752048   789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b7a634cdcdbb46aa9d3f79b834ea7d37. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:40.767150   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 67776.7us
I20250628 07:52:40.767973  2392 consensus_queue.cc:1035] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:40.768281  1229 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:40.769153  1229 ts_tablet_manager.cc:1397] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.073s	user 0.007s	sys 0.013s
I20250628 07:52:40.774897  1229 raft_consensus.cc:357] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.776841  1229 raft_consensus.cc:383] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.778285  1229 raft_consensus.cc:738] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:40.781850  2512 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20250628 07:52:40.781424  1229 consensus_queue.cc:260] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.789417  2428 ts_tablet_manager.cc:1918] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:40.790958  2428 log.cc:1199] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/de7ff3260b0049d682dba82f914474de
I20250628 07:52:40.792456  1229 ts_tablet_manager.cc:1428] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.023s	user 0.008s	sys 0.001s
I20250628 07:52:40.794020  2428 ts_tablet_manager.cc:1939] T de7ff3260b0049d682dba82f914474de P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:40.797048  1229 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:40.807446  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 382) took 1576 ms. Trace:
I20250628 07:52:40.809172  2428 rpcz_store.cc:276] 0628 07:52:39.230641 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.230749 (+   108us) service_pool.cc:225] Handling call
0628 07:52:40.690739 (+1459990us) ts_tablet_manager.cc:1145] Deleting tablet de7ff3260b0049d682dba82f914474de
0628 07:52:40.690779 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.729154 (+ 38375us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.788498 (+ 59344us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.807151 (+ 18653us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1459495}
I20250628 07:52:40.818415  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 384) took 1384 ms. Trace:
I20250628 07:52:40.819844  2428 rpcz_store.cc:276] 0628 07:52:39.432262 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.435572 (+  3310us) service_pool.cc:225] Handling call
0628 07:52:40.814215 (+1378643us) ts_tablet_manager.cc:1145] Deleting tablet d2578fbf2e974cc0bca65e32f45694ab
0628 07:52:40.814374 (+   159us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.816766 (+  2392us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1377863}
I20250628 07:52:40.826467  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 383) took 1441 ms. Trace:
I20250628 07:52:40.830953   965 tablet_service.cc:1468] Processing CreateTablet for tablet ca128104712e4b98b6bea2cb4bd604f2 (DEFAULT_TABLE table=default.Table_168a40d10a3645bdaea410044e99f357 [id=2013562180c34579a816dde8278e9c85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.831887   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ca128104712e4b98b6bea2cb4bd604f2. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:40.833101   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 6fdda1e4f7af468982a813f1ac023e01 (table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 6fdda1e4f7af468982a813f1ac023e01 consensus state has no leader
I20250628 07:52:40.833284   954 tablet_service.cc:1515] Processing DeleteTablet for tablet d87dd96d00964e12acd6e380a62c94d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:40.837858   546 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:47964 (request call id 427) took 1503 ms. Trace:
W20250628 07:52:40.840487   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 72710.1us
W20250628 07:52:40.841164   921 leader_election.cc:343] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:40.838527   546 rpcz_store.cc:276] 0628 07:52:39.333514 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.333702 (+   188us) service_pool.cc:225] Handling call
0628 07:52:39.334603 (+   901us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:39.335224 (+   621us) catalog_manager.cc:3683] Apply alter schema
0628 07:52:39.335746 (+   522us) catalog_manager.cc:3715] Acquired catalog manager lock
0628 07:52:39.335938 (+   192us) catalog_manager.cc:3873] Updating metadata on disk
0628 07:52:39.352691 (+ 16753us) catalog_manager.cc:3904] Committing alterations to in-memory state
0628 07:52:40.837161 (+1484470us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:39.337571 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:39.338068 (+   497us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:39.338096 (+    28us) write_op.cc:435] Acquired schema lock
0628 07:52:39.338106 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:39.338340 (+   234us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:39.338377 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:39.338398 (+    21us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:39.338542 (+   144us) tablet.cc:699] Row locks acquired
0628 07:52:39.338574 (+    32us) write_op.cc:260] PREPARE: finished
0628 07:52:39.338707 (+   133us) write_op.cc:270] Start()
0628 07:52:39.338805 (+    98us) write_op.cc:276] Timestamp: P: 1751097159338536 usec, L: 0
0628 07:52:39.338824 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:39.339391 (+   567us) log.cc:844] Serialized 405 byte log entry
0628 07:52:39.346191 (+  6800us) op_driver.cc:464] REPLICATION: finished
0628 07:52:39.346491 (+   300us) write_op.cc:301] APPLY: starting
0628 07:52:39.346544 (+    53us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:39.346640 (+    96us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:39.346729 (+    89us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:39.346895 (+   166us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:39.347027 (+   132us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:39.347079 (+    52us) write_op.cc:312] APPLY: finished
0628 07:52:39.347244 (+   165us) log.cc:844] Serialized 23 byte log entry
0628 07:52:39.347381 (+   137us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:39.347439 (+    58us) write_op.cc:454] Released schema lock
0628 07:52:39.347475 (+    36us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:39.347684 (+   209us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:39.347714 (+    30us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":170,"apply.run_cpu_time_us":1161,"apply.run_wall_time_us":1158,"num_ops":1,"prepare.queue_time_us":103,"prepare.run_cpu_time_us":2119,"prepare.run_wall_time_us":3389,"raft.queue_time_us":74,"raft.run_cpu_time_us":496,"raft.run_wall_time_us":496,"replication_time_us":7279,"spinlock_wait_cycles":19712}]]}
I20250628 07:52:40.845247   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]) successfully deleted
I20250628 07:52:40.846140   546 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_753078c239dc43a6a5125732a2ff887c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:40.847059   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet e536268f648d4b0da1b9edc338ae974a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e536268f648d4b0da1b9edc338ae974a already in progress: creating tablet
I20250628 07:52:40.847978   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet de7ff3260b0049d682dba82f914474de (table default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]) successfully deleted
W20250628 07:52:40.859441   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 65) took 2778 ms (client timeout 1000 ms). Trace:
W20250628 07:52:40.859769   541 rpcz_store.cc:269] 0628 07:52:38.081266 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.130063 (+ 48797us) service_pool.cc:225] Handling call
0628 07:52:40.859255 (+2729192us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":22562,"reported_tablets":37}
I20250628 07:52:40.861500   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3b7ef98c85564dc9b40201e1713b7c55"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:40.811770   540 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 64) took 3821 ms (client timeout 1000 ms). Trace:
W20250628 07:52:40.863379   540 rpcz_store.cc:269] 0628 07:52:36.913013 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.920870 (+  7857us) service_pool.cc:225] Handling call
0628 07:52:40.734363 (+3813493us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:37.544713 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:37.545135 (+   422us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:37.545158 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:37.545171 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:37.545445 (+   274us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:37.545481 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:37.545501 (+    20us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:37.545736 (+   235us) tablet.cc:699] Row locks acquired
0628 07:52:37.545749 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:37.545859 (+   110us) write_op.cc:270] Start()
0628 07:52:37.545929 (+    70us) write_op.cc:276] Timestamp: P: 1751097157545786 usec, L: 0
0628 07:52:37.545944 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:37.546350 (+   406us) log.cc:844] Serialized 913 byte log entry
0628 07:52:37.583130 (+ 36780us) op_driver.cc:464] REPLICATION: finished
0628 07:52:37.583842 (+   712us) write_op.cc:301] APPLY: starting
0628 07:52:37.583898 (+    56us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:37.583996 (+    98us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:37.584088 (+    92us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:37.584332 (+   244us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:37.584453 (+   121us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:37.584494 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:37.584651 (+   157us) log.cc:844] Serialized 29 byte log entry
0628 07:52:37.584787 (+   136us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:37.584855 (+    68us) write_op.cc:454] Released schema lock
0628 07:52:37.584885 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:37.585080 (+   195us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:37.585119 (+    39us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":225838,"reported_tablets":37,"child_traces":[["op",{"apply.queue_time_us":580,"apply.run_cpu_time_us":1214,"apply.run_wall_time_us":1215,"num_ops":2,"prepare.queue_time_us":83,"prepare.run_cpu_time_us":1810,"prepare.run_wall_time_us":1810,"replication_time_us":37102}]]}
W20250628 07:52:40.848523   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d2578fbf2e974cc0bca65e32f45694ab because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d2578fbf2e974cc0bca65e32f45694ab already in progress: creating tablet
I20250628 07:52:40.828044  2428 rpcz_store.cc:276] 0628 07:52:39.384494 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.421673 (+ 37179us) service_pool.cc:225] Handling call
0628 07:52:40.824318 (+1402645us) ts_tablet_manager.cc:1145] Deleting tablet 35fe2e8156cc43d7b3c02092877063b6
0628 07:52:40.824626 (+   308us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.826300 (+  1674us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1385676}
I20250628 07:52:40.876473  1229 tablet_bootstrap.cc:654] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.871723   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_cc2df6969d8d43ba882f1832a7c4b528"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:40.635851  2237 ts_tablet_manager.cc:1918] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:40.636204   824 raft_consensus.cc:2466] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 2.
I20250628 07:52:40.643118   792 tablet_service.cc:1468] Processing CreateTablet for tablet ca128104712e4b98b6bea2cb4bd604f2 (DEFAULT_TABLE table=default.Table_168a40d10a3645bdaea410044e99f357 [id=2013562180c34579a816dde8278e9c85]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.882987   792 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ca128104712e4b98b6bea2cb4bd604f2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.885762   920 leader_election.cc:304] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4; no voters: 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:40.894728   791 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.898784   789 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:40.903421  2237 log.cc:1199] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/5fa4299a9e0f4b409d04c8c5900901c0
I20250628 07:52:40.904243  2237 ts_tablet_manager.cc:1939] T 5fa4299a9e0f4b409d04c8c5900901c0 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:40.906275  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 336) took 4748 ms. Trace:
I20250628 07:52:40.913003  1100 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 21ee5fbf572c4530939c1c103b89737e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.914673  2410 raft_consensus.cc:491] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.915112  2410 raft_consensus.cc:513] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.919365  1230 raft_consensus.cc:357] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.919999  1230 raft_consensus.cc:383] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:40.920336  1230 raft_consensus.cc:738] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:40.881208   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 59) took 2044 ms (client timeout 1000 ms). Trace:
W20250628 07:52:40.923198   547 rpcz_store.cc:269] 0628 07:52:38.836869 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.836986 (+   117us) service_pool.cc:225] Handling call
0628 07:52:38.847407 (+ 10421us) spinlock_profiling.cc:243] Waited 6.77 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:40.881065 (+2033658us) inbound_call.cc:173] Queueing success response
Metrics: {"reported_tablets":21,"spinlock_wait_cycles":15576832}
I20250628 07:52:40.924155   547 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_df287bcea5bd414ebe5aee1689733f8c" } modify_external_catalogs: true
I20250628 07:52:40.924657   547 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_df287bcea5bd414ebe5aee1689733f8c" } modify_external_catalogs: true
W20250628 07:52:40.875056   546 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_753078c239dc43a6a5125732a2ff887c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:40.896364   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 58) took 3907 ms (client timeout 1000 ms). Trace:
W20250628 07:52:40.929437   538 rpcz_store.cc:269] 0628 07:52:36.988538 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.485935 (+497397us) service_pool.cc:225] Handling call
0628 07:52:40.896194 (+3410259us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:37.982441 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:37.982800 (+   359us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:37.982822 (+    22us) write_op.cc:435] Acquired schema lock
0628 07:52:37.982833 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:37.983105 (+   272us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:37.983138 (+    33us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:37.983157 (+    19us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:37.983391 (+   234us) tablet.cc:699] Row locks acquired
0628 07:52:37.983403 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:37.983499 (+    96us) write_op.cc:270] Start()
0628 07:52:37.983572 (+    73us) write_op.cc:276] Timestamp: P: 1751097157983437 usec, L: 0
0628 07:52:37.983586 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:37.984034 (+   448us) log.cc:844] Serialized 913 byte log entry
0628 07:52:38.170604 (+186570us) op_driver.cc:464] REPLICATION: finished
0628 07:52:38.170889 (+   285us) write_op.cc:301] APPLY: starting
0628 07:52:38.170935 (+    46us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:38.171036 (+   101us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:38.171111 (+    75us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:38.171324 (+   213us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:38.171445 (+   121us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:38.171485 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:38.171673 (+   188us) log.cc:844] Serialized 29 byte log entry
0628 07:52:38.171795 (+   122us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.171848 (+    53us) write_op.cc:454] Released schema lock
0628 07:52:38.171884 (+    36us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:38.172062 (+   178us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:38.172083 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":8828,"reported_tablets":18,"child_traces":[["op",{"apply.queue_time_us":161,"apply.run_cpu_time_us":1148,"apply.run_wall_time_us":1147,"num_ops":2,"prepare.queue_time_us":97,"prepare.run_cpu_time_us":1828,"prepare.run_wall_time_us":2801,"replication_time_us":186927}]]}
I20250628 07:52:40.935154   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_bb662e3441d74b638d7c33dc69755650"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:40.876276   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3b7ef98c85564dc9b40201e1713b7c55 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:40.877403  2428 tablet_replica.cc:331] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
W20250628 07:52:40.938097   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bb662e3441d74b638d7c33dc69755650 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:40.940924   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_cc2df6969d8d43ba882f1832a7c4b528 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:40.952950   963 tablet_service.cc:1468] Processing CreateTablet for tablet b7a634cdcdbb46aa9d3f79b834ea7d37 (DEFAULT_TABLE table=default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4 [id=3594565eae314c6e861c681fb61092bf]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.953977   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b7a634cdcdbb46aa9d3f79b834ea7d37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.954787  1091 tablet_service.cc:1468] Processing CreateTablet for tablet 586f1b7dd30e4998abebaa1fb2983e2b (DEFAULT_TABLE table=default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.956075  2103 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "6005c44dfe4848f18a304621cd49e934" instance_seqno: 1751097126033738) as {username='slave'} at 127.0.107.129:39311; Asking this server to re-register.
I20250628 07:52:40.916971  2410 leader_election.cc:290] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:40.962576  2428 raft_consensus.cc:2241] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:40.963132  2428 raft_consensus.cc:2270] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:40.917786   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:40.966648  2237 rpcz_store.cc:276] 0628 07:52:36.158142 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.158244 (+   102us) service_pool.cc:225] Handling call
0628 07:52:39.971233 (+3812989us) ts_tablet_manager.cc:1145] Deleting tablet 5fa4299a9e0f4b409d04c8c5900901c0
0628 07:52:39.991107 (+ 19874us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.187503 (+196396us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.635664 (+448161us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:40.906216 (+270552us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":19356,"tablet-delete.queue_time_us":3812455}
W20250628 07:52:40.967064  1062 leader_election.cc:343] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:40.978025   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 5fa4299a9e0f4b409d04c8c5900901c0 (table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]) successfully deleted
I20250628 07:52:40.981173   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:40.983084   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:40.921048  1230 consensus_queue.cc:260] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:40.987206   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:40.987615  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 337) took 4451 ms. Trace:
I20250628 07:52:40.987905  2237 rpcz_store.cc:276] 0628 07:52:36.536306 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.536411 (+   105us) service_pool.cc:225] Handling call
0628 07:52:40.987232 (+4450821us) ts_tablet_manager.cc:1145] Deleting tablet af28ae2139b2417381808cb4068ddf60
0628 07:52:40.987283 (+    51us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.987553 (+   270us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4436209}
I20250628 07:52:40.988656   546 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 430) took 1558 ms. Trace:
I20250628 07:52:40.989008   541 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 431) took 1499 ms. Trace:
W20250628 07:52:40.989110   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet af28ae2139b2417381808cb4068ddf60 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet af28ae2139b2417381808cb4068ddf60 already in progress: creating tablet
I20250628 07:52:40.974397  1234 tablet_bootstrap.cc:654] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:40.991880  1096 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3d1bc8483374a2a917451a43b3d42ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.988925   546 rpcz_store.cc:276] 0628 07:52:39.430247 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.845784 (+1415537us) service_pool.cc:225] Handling call
0628 07:52:40.928420 (+ 82636us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:40.956180 (+ 27760us) spinlock_profiling.cc:243] Waited 19.9 ms on lock 0x7b0c00033dd8. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be1a869b1 00007f5be1b9399f 00007f5be1b937d9 00007f5bea8236df 00007f5bea821391 00007f5bea81ebdd 00007f5bea82256e 00007f5bea940d6f 00007f5be865b6a4 00007f5be865b632 00007f5be865b561
0628 07:52:40.956813 (+   633us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:40.988079 (+ 31266us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:40.988347 (+   268us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:40.988575 (+   228us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.964098 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.964589 (+   491us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.964622 (+    33us) write_op.cc:435] Acquired schema lock
0628 07:52:40.964643 (+    21us) tablet.cc:660] Decoding operations
0628 07:52:40.965008 (+   365us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.965055 (+    47us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.965107 (+    52us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.965380 (+   273us) tablet.cc:699] Row locks acquired
0628 07:52:40.965401 (+    21us) write_op.cc:260] PREPARE: finished
0628 07:52:40.965496 (+    95us) write_op.cc:270] Start()
0628 07:52:40.965569 (+    73us) write_op.cc:276] Timestamp: P: 1751097160965447 usec, L: 0
0628 07:52:40.965582 (+    13us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.966006 (+   424us) log.cc:844] Serialized 433 byte log entry
0628 07:52:40.983863 (+ 17857us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.986010 (+  2147us) write_op.cc:301] APPLY: starting
0628 07:52:40.986059 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.986148 (+    89us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.986245 (+    97us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.986704 (+   459us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.986830 (+   126us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:40.986872 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:40.987563 (+   691us) log.cc:844] Serialized 29 byte log entry
0628 07:52:40.987697 (+   134us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.987750 (+    53us) write_op.cc:454] Released schema lock
0628 07:52:40.987776 (+    26us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":45748096,"child_traces":[["op",{"apply.queue_time_us":2027,"apply.run_cpu_time_us":1932,"apply.run_wall_time_us":1932,"mutex_wait_us":6130,"num_ops":2,"prepare.queue_time_us":6271,"prepare.run_cpu_time_us":2090,"prepare.run_wall_time_us":3124,"replication_time_us":18206,"threads_started":1}]]}
I20250628 07:52:40.989405   541 rpcz_store.cc:276] 0628 07:52:39.489549 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.860968 (+1371419us) service_pool.cc:225] Handling call
0628 07:52:40.936268 (+ 75300us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:40.953981 (+ 17713us) spinlock_profiling.cc:243] Waited 11.8 ms on lock 0x7b0c00033dd8. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be1a869b1 00007f5be1b9399f 00007f5be1b937d9 00007f5bea8236df 00007f5bea821391 00007f5bea81ebdd 00007f5bea82256e 00007f5bea940d6f 00007f5be865b6a4 00007f5be865b632 00007f5be865b561
0628 07:52:40.954735 (+   754us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:40.988414 (+ 33679us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:40.988706 (+   292us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:40.988937 (+   231us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.958787 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.959159 (+   372us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.959184 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:40.959198 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:40.959580 (+   382us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.959618 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.959642 (+    24us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.959861 (+   219us) tablet.cc:699] Row locks acquired
0628 07:52:40.959877 (+    16us) write_op.cc:260] PREPARE: finished
0628 07:52:40.959987 (+   110us) write_op.cc:270] Start()
0628 07:52:40.960070 (+    83us) write_op.cc:276] Timestamp: P: 1751097160959934 usec, L: 0
0628 07:52:40.960085 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.961126 (+  1041us) log.cc:844] Serialized 433 byte log entry
0628 07:52:40.983657 (+ 22531us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.984443 (+   786us) write_op.cc:301] APPLY: starting
0628 07:52:40.984491 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.984577 (+    86us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.984648 (+    71us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.985013 (+   365us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.985146 (+   133us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:40.985187 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:40.985327 (+   140us) log.cc:844] Serialized 29 byte log entry
0628 07:52:40.985451 (+   124us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.985507 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:40.985535 (+    28us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:40.985766 (+   231us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.985790 (+    24us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":27097984,"child_traces":[["op",{"apply.queue_time_us":639,"apply.run_cpu_time_us":1292,"apply.run_wall_time_us":1293,"num_ops":2,"prepare.queue_time_us":2937,"prepare.run_cpu_time_us":2495,"prepare.run_wall_time_us":5004,"replication_time_us":23474}]]}
I20250628 07:52:40.996650  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 338) took 4094 ms. Trace:
I20250628 07:52:40.828876   958 tablet_service.cc:1468] Processing CreateTablet for tablet 4d21243664ea4bbeb4a7c31d8c019486 (DEFAULT_TABLE table=default.Table_8a1536aea0c64577a92168afbd496c5c [id=5bbb2a3c166044139f30f8650e698936]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:40.997294   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:40.996515   541 master_service.cc:948] ReplaceTablet: received request to replace tablet c4fcefaa9e954b52bbbee6e0a2f2f939 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:41.001709  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a4900b15c92a4024917480a3fce8373c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.003108  2059 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:41.003868  2328 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:41.006127  1093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f42a3a559b448f828d724383043a12. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.009558  1095 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33ae7e653d644224a07a3bf29ef9fcd9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.010195  1097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c932ba2738a414da4405677d886dabe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:40.951268  2432 raft_consensus.cc:491] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:40.938037  2392 raft_consensus.cc:2802] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Leader election won for term 2
I20250628 07:52:41.017165  2237 rpcz_store.cc:276] 0628 07:52:36.902402 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:36.902511 (+   109us) service_pool.cc:225] Handling call
0628 07:52:40.996310 (+4093799us) ts_tablet_manager.cc:1145] Deleting tablet f0c00063b47e49099a2c0ab9c8587c04
0628 07:52:40.996348 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:40.996600 (+   252us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4093254}
I20250628 07:52:41.027734  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6ce66199d1614826bf618819f6928383. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.029902  1098 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1da78a71a634d408fb5de4efe92ebbe. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.031764  1099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec358ab4881d42ad8ffbcf3464ef59a3. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:41.033737   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet f0c00063b47e49099a2c0ab9c8587c04 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f0c00063b47e49099a2c0ab9c8587c04 already in progress: creating tablet
I20250628 07:52:41.034623  2237 tablet_replica.cc:331] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:41.036414  2237 raft_consensus.cc:2241] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:41.035213   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]). Expected version 2 got 0
I20250628 07:52:41.036965  2237 raft_consensus.cc:2270] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:41.037124   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet bfc303c4588d44428984d8203fc05185 (table default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]). Expected version 1 got 0
I20250628 07:52:41.037700  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cc49fd5696e1403e95ea8d60220eafa0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.039513  1092 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b1fcc569fb1b4a84983b651074e3e64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.040056   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:41.042351   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:41.043192  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f49a0de38fe24baab09728a19513878f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.044539  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06b9b63649384313948964b3c0dd11cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.045738  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 64f5ca605ce4480da9eaad08456ddb9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.049580  1104 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3cfb8eefaa45432bb50cc729a401a2ec. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.053638  1109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d21243664ea4bbeb4a7c31d8c019486. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.069553  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 80ee935a2d58422fb1bfbc9103940920. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.074415  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b7a634cdcdbb46aa9d3f79b834ea7d37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.077466  2514 tablet_replica.cc:331] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:41.078611  1090 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ca128104712e4b98b6bea2cb4bd604f2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.029347   874 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:41.082993  1230 ts_tablet_manager.cc:1428] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.367s	user 0.004s	sys 0.004s
I20250628 07:52:41.089083   543 catalog_manager.cc:5582] T 0b251095244548629ae93d52e6c80ccc P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:41.092293  2428 ts_tablet_manager.cc:1905] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:41.093947  2237 ts_tablet_manager.cc:1905] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:41.016862  2432 raft_consensus.cc:513] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:41.117225   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 113846us
I20250628 07:52:41.120328  2428 ts_tablet_manager.cc:1918] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:41.120661  2428 log.cc:1199] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/d87dd96d00964e12acd6e380a62c94d8
I20250628 07:52:41.121469  2428 ts_tablet_manager.cc:1939] T d87dd96d00964e12acd6e380a62c94d8 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:41.134222  2103 ts_manager.cc:194] Registered new tserver with Master: 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:41.138350  1234 tablet_bootstrap.cc:492] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:41.138841  1234 ts_tablet_manager.cc:1397] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.409s	user 0.007s	sys 0.006s
I20250628 07:52:41.142933  1234 raft_consensus.cc:357] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.148170  1229 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:41.148581  1229 ts_tablet_manager.cc:1397] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.353s	user 0.008s	sys 0.014s
I20250628 07:52:41.161316  1234 raft_consensus.cc:383] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:41.162089  1234 raft_consensus.cc:738] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:41.167436  1234 consensus_queue.cc:260] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.172870   540 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 432) took 1661 ms. Trace:
I20250628 07:52:41.173164   540 rpcz_store.cc:276] 0628 07:52:39.511485 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.866788 (+1355303us) service_pool.cc:225] Handling call
0628 07:52:40.945400 (+ 78612us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:40.971126 (+ 25726us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:41.134696 (+163570us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:41.172518 (+ 37822us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:41.172781 (+   263us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.975420 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.975793 (+   373us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.975819 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:40.975830 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:40.976172 (+   342us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.976226 (+    54us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.976262 (+    36us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.976527 (+   265us) tablet.cc:699] Row locks acquired
0628 07:52:40.976547 (+    20us) write_op.cc:260] PREPARE: finished
0628 07:52:40.976692 (+   145us) write_op.cc:270] Start()
0628 07:52:40.976812 (+   120us) write_op.cc:276] Timestamp: P: 1751097160976627 usec, L: 0
0628 07:52:40.976835 (+    23us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.977684 (+   849us) log.cc:844] Serialized 433 byte log entry
0628 07:52:41.131027 (+153343us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.131465 (+   438us) write_op.cc:301] APPLY: starting
0628 07:52:41.131527 (+    62us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:41.131621 (+    94us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:41.131709 (+    88us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:41.132138 (+   429us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:41.132299 (+   161us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:41.132344 (+    45us) write_op.cc:312] APPLY: finished
0628 07:52:41.132509 (+   165us) log.cc:844] Serialized 29 byte log entry
0628 07:52:41.132649 (+   140us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.132713 (+    64us) write_op.cc:454] Released schema lock
0628 07:52:41.132743 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:41.132983 (+   240us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.133005 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":277,"apply.run_cpu_time_us":1485,"apply.run_wall_time_us":1485,"num_ops":2,"prepare.queue_time_us":3225,"prepare.run_cpu_time_us":2144,"prepare.run_wall_time_us":3282,"replication_time_us":154086}]]}
I20250628 07:52:41.176482   540 master_service.cc:948] ReplaceTablet: received request to replace tablet af13b56df157443c994f747e52b0f458 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:41.177353   538 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:47964 (request call id 434) took 1560 ms. Trace:
I20250628 07:52:41.178124  1234 ts_tablet_manager.cc:1428] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.037s	user 0.005s	sys 0.000s
I20250628 07:52:41.179070  1234 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:41.177698   538 rpcz_store.cc:276] 0628 07:52:39.616573 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.934841 (+1318268us) service_pool.cc:225] Handling call
0628 07:52:40.939002 (+  4161us) catalog_manager.cc:2076] Acquired catalog manager lock
0628 07:52:40.958096 (+ 19094us) spinlock_profiling.cc:243] Waited 9.39 ms on lock 0x7b0c00033dd8. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be1a869b1 00007f5be1b9399f 00007f5be1b937d9 00007f5bea8236df 00007f5bea821391 00007f5bea81ebdd 00007f5bea82256e 00007f5bea940d6f 00007f5be865b6a4 00007f5be865b632 00007f5be865b561
0628 07:52:40.972435 (+ 14339us) catalog_manager.cc:2122] Created new table and tablet info
0628 07:52:41.134419 (+161984us) catalog_manager.cc:2182] Wrote table and tablets to system table
0628 07:52:41.176997 (+ 42578us) catalog_manager.cc:2216] Inserted table and tablets into CatalogManager maps
0628 07:52:41.177285 (+   288us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.978813 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.979191 (+   378us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.979214 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:40.979226 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:40.979482 (+   256us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.979519 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.979542 (+    23us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.979755 (+   213us) tablet.cc:699] Row locks acquired
0628 07:52:40.979777 (+    22us) write_op.cc:260] PREPARE: finished
0628 07:52:40.979886 (+   109us) write_op.cc:270] Start()
0628 07:52:40.979964 (+    78us) write_op.cc:276] Timestamp: P: 1751097160979832 usec, L: 0
0628 07:52:40.979979 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.981113 (+  1134us) log.cc:844] Serialized 433 byte log entry
0628 07:52:41.131260 (+150147us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.133233 (+  1973us) write_op.cc:301] APPLY: starting
0628 07:52:41.133278 (+    45us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:41.133357 (+    79us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:41.133426 (+    69us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:41.133759 (+   333us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:41.133880 (+   121us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
0628 07:52:41.133914 (+    34us) write_op.cc:312] APPLY: finished
0628 07:52:41.134048 (+   134us) log.cc:844] Serialized 29 byte log entry
0628 07:52:41.134161 (+   113us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.134216 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:41.134246 (+    30us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":21598976,"child_traces":[["op",{"apply.queue_time_us":1857,"apply.run_cpu_time_us":1187,"apply.run_wall_time_us":1186,"mutex_wait_us":3800,"num_ops":2,"prepare.queue_time_us":5240,"prepare.run_cpu_time_us":2472,"prepare.run_wall_time_us":3205,"replication_time_us":151208}]]}
I20250628 07:52:41.195549  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 586f1b7dd30e4998abebaa1fb2983e2b. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:41.203245  2470 log.cc:927] Time spent T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Append to log took a long time: real 1.516s	user 0.029s	sys 0.013s
I20250628 07:52:41.222270  2536 raft_consensus.cc:491] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:41.222806  2536 raft_consensus.cc:513] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.210042  2410 raft_consensus.cc:491] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:41.229800  2410 raft_consensus.cc:513] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.231549  2410 leader_election.cc:290] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:41.234869  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:41.235356  1124 raft_consensus.cc:2466] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:41.239058   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_00d339485c8d4aa38db91013d26fdcc5" } modify_external_catalogs: true
I20250628 07:52:41.239495   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_00d339485c8d4aa38db91013d26fdcc5" } modify_external_catalogs: true
I20250628 07:52:41.240001  2536 leader_election.cc:290] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:41.244581   957 tablet_service.cc:1468] Processing CreateTablet for tablet 586f1b7dd30e4998abebaa1fb2983e2b (DEFAULT_TABLE table=default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:41.245994   921 leader_election.cc:304] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:41.250164  1230 tablet_bootstrap.cc:492] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:41.083801  2402 consensus_queue.cc:1035] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.037s
I20250628 07:52:41.253684   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
I20250628 07:52:41.254292   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:41.255075   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]). Expected version 1 got 0
I20250628 07:52:41.256054   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:41.263108   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bfc303c4588d44428984d8203fc05185 (table default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]). Expected version 1 got 0
I20250628 07:52:41.262295  2538 raft_consensus.cc:2802] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:41.264501  2538 raft_consensus.cc:491] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:41.264837  2538 raft_consensus.cc:3058] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:41.264081   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:41.268543   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fd67d7d9ba9446c19f1711fb5bd73ebd" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:41.269593   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 2 got 1
I20250628 07:52:41.272920   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:41.278668   920 leader_election.cc:343] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:41.280356  1062 leader_election.cc:343] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.284775  1230 tablet_bootstrap.cc:654] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:41.304126  2514 raft_consensus.cc:2241] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:41.304698  2514 raft_consensus.cc:2270] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:41.305716   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet af13b56df157443c994f747e52b0f458
I20250628 07:52:41.306950  2514 ts_tablet_manager.cc:1905] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:41.307180   792 tablet_service.cc:1515] Processing DeleteTablet for tablet af13b56df157443c994f747e52b0f458 with delete_type TABLET_DATA_DELETED (replaced by tablet f2295f958cb34b658d2dfcf836d25e7f) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.243870  2410 raft_consensus.cc:491] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:41.310336   540 catalog_manager.cc:6652] ReplaceTablet: tablet af13b56df157443c994f747e52b0f458 deleted and replaced by tablet f2295f958cb34b658d2dfcf836d25e7f
I20250628 07:52:41.313943   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet c4fcefaa9e954b52bbbee6e0a2f2f939
I20250628 07:52:41.315249   792 tablet_service.cc:1515] Processing DeleteTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 with delete_type TABLET_DATA_DELETED (replaced by tablet 4eac11942d104acab461d483252128c5) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.316710   541 catalog_manager.cc:6652] ReplaceTablet: tablet c4fcefaa9e954b52bbbee6e0a2f2f939 deleted and replaced by tablet 4eac11942d104acab461d483252128c5
I20250628 07:52:41.317040   541 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:47964 (request call id 435) took 1221 ms. Trace:
I20250628 07:52:41.319181  2410 raft_consensus.cc:513] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.317409   541 rpcz_store.cc:276] 0628 07:52:40.095957 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.996403 (+900446us) service_pool.cc:225] Handling call
0628 07:52:41.140187 (+143784us) spinlock_profiling.cc:243] Waited 129 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea943891 00007f5be867c3a7 00007f5be867c332 00007f5be867c261 00007f5be867c1dc 00007f5be867b492 00007f5be6957dcc 00007f5be6957206
0628 07:52:41.297420 (+157233us) spinlock_profiling.cc:243] Waited 6.78 ms on lock 0x7b54000023e8. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be1a869b1 00007f5be1be601e 00007f5bea8643a7 00007f5bea8435cc 00007f5bea9438d4 00007f5be867c3a7 00007f5be867c332 00007f5be867c261 00007f5be867c1dc 00007f5be867b492 00007f5be6957dcc
0628 07:52:41.316968 (+ 19548us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:41.299381 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:41.299803 (+   422us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:41.299830 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:41.299848 (+    18us) tablet.cc:660] Decoding operations
0628 07:52:41.300172 (+   324us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:41.300210 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:41.300231 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:41.300433 (+   202us) tablet.cc:699] Row locks acquired
0628 07:52:41.300445 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:41.300565 (+   120us) write_op.cc:270] Start()
0628 07:52:41.300645 (+    80us) write_op.cc:276] Timestamp: P: 1751097161300476 usec, L: 0
0628 07:52:41.300713 (+    68us) op_driver.cc:348] REPLICATION: starting
0628 07:52:41.301221 (+   508us) log.cc:844] Serialized 662 byte log entry
0628 07:52:41.310829 (+  9608us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.311049 (+   220us) write_op.cc:301] APPLY: starting
0628 07:52:41.311098 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:41.311187 (+    89us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:41.311260 (+    73us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:41.311678 (+   418us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:41.311853 (+   175us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:41.311892 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:41.312063 (+   171us) log.cc:844] Serialized 29 byte log entry
0628 07:52:41.312180 (+   117us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.312237 (+    57us) write_op.cc:454] Released schema lock
0628 07:52:41.312263 (+    26us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:41.313185 (+   922us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.313206 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":311511168,"child_traces":[["op",{"apply.queue_time_us":83,"apply.run_cpu_time_us":1415,"apply.run_wall_time_us":2103,"num_ops":2,"prepare.queue_time_us":141,"prepare.run_cpu_time_us":1990,"prepare.run_wall_time_us":2910,"replication_time_us":10021}]]}
I20250628 07:52:41.322328  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 with delete_type TABLET_DATA_DELETED (replaced by tablet 4eac11942d104acab461d483252128c5) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.322223   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_fc4c2c1594ad4e17ab99e4e2eee626d4" } new_table_name: "default.Table_e13a07ffb7754b44a1e224a0cc1e4289" modify_external_catalogs: true
I20250628 07:52:41.322328  1097 tablet_service.cc:1515] Processing DeleteTablet for tablet af13b56df157443c994f747e52b0f458 with delete_type TABLET_DATA_DELETED (replaced by tablet f2295f958cb34b658d2dfcf836d25e7f) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.326555   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:41.320894  2410 leader_election.cc:290] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:41.328473  1062 leader_election.cc:343] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.330740   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_b68d37d5ffd14e17820af67d5f0d7a71" } new_table_name: "default.Table_00cd764ff56846b289203b0111658952" modify_external_catalogs: true
I20250628 07:52:41.342598   961 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.373642  1096 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 373) took 1101 ms. Trace:
I20250628 07:52:41.377539  1096 rpcz_store.cc:276] 0628 07:52:40.272719 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.272956 (+   237us) service_pool.cc:225] Handling call
0628 07:52:40.984597 (+711641us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:40.991472 (+  6875us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.001024 (+  9552us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.317571 (+316547us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.344490 (+ 26919us) spinlock_profiling.cc:243] Waited 22.9 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b4512 00007fbaa7d527e9 00007fbaa6fb2968 00007fbaa7077083 00007fbaa7072e4f 00007fbaa707360d 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2
0628 07:52:41.373562 (+ 29072us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":52574976}
W20250628 07:52:41.379272   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet b7a634cdcdbb46aa9d3f79b834ea7d37 consensus state has no leader
I20250628 07:52:41.380442   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_e20c49ea9aae4980b56ee14957a8c928"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.388980   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e20c49ea9aae4980b56ee14957a8c928 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.391358  2237 ts_tablet_manager.cc:1918] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20250628 07:52:41.395382  2237 log.cc:1199] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:41.397018  2237 ts_tablet_manager.cc:1939] T e15d063a4bca40d480cba08733ec1ec3 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:41.406551   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_14ad18ef5e224208bffdbc4a88018111"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.407887   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_14ad18ef5e224208bffdbc4a88018111 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.410949  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 350) took 3912 ms. Trace:
I20250628 07:52:41.416569   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 06b9b63649384313948964b3c0dd11cf
I20250628 07:52:41.417205  2237 rpcz_store.cc:276] 0628 07:52:37.498320 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.532035 (+ 33715us) service_pool.cc:225] Handling call
0628 07:52:41.034435 (+3502400us) ts_tablet_manager.cc:1145] Deleting tablet e15d063a4bca40d480cba08733ec1ec3
0628 07:52:41.034488 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.190751 (+156263us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.368288 (+177537us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.410883 (+ 42595us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3492870}
I20250628 07:52:41.419241  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 363) took 3678 ms. Trace:
I20250628 07:52:41.419909   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet e15d063a4bca40d480cba08733ec1ec3 (table default.Table_52441cbec305498ebed774e69004a383 [id=475e75d866ea4a5585227052d39db9da]) successfully deleted
I20250628 07:52:41.418393   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_0f6f46bde0ed463eb196dfdf0f049317"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:41.421207   792 tablet_service.cc:1515] Processing DeleteTablet for tablet 06b9b63649384313948964b3c0dd11cf with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.422209  2237 rpcz_store.cc:276] 0628 07:52:37.740319 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.740427 (+   108us) service_pool.cc:225] Handling call
0628 07:52:41.418953 (+3678526us) ts_tablet_manager.cc:1145] Deleting tablet 0dffb3eb778946869b914e5b05a67a69
0628 07:52:41.419000 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.419185 (+   185us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3678001}
I20250628 07:52:41.423491  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 364) took 3682 ms. Trace:
W20250628 07:52:41.424289   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 0dffb3eb778946869b914e5b05a67a69 because the tablet was not found. No further retry: Not found: Tablet not found: 0dffb3eb778946869b914e5b05a67a69
W20250628 07:52:41.425010   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0f6f46bde0ed463eb196dfdf0f049317 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.424685  2237 rpcz_store.cc:276] 0628 07:52:37.740600 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.740696 (+    96us) service_pool.cc:225] Handling call
0628 07:52:41.423205 (+3682509us) ts_tablet_manager.cc:1145] Deleting tablet cc0bf0edebb140209531ccf917fd1ed8
0628 07:52:41.423252 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.423417 (+   165us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3681990}
W20250628 07:52:41.430639   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet cc0bf0edebb140209531ccf917fd1ed8 because the tablet was not found. No further retry: Not found: Tablet not found: cc0bf0edebb140209531ccf917fd1ed8
I20250628 07:52:41.430569  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 365) took 3689 ms. Trace:
I20250628 07:52:41.431871  2237 rpcz_store.cc:276] 0628 07:52:37.740880 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.740999 (+   119us) service_pool.cc:225] Handling call
0628 07:52:41.430288 (+3689289us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:41.430336 (+    48us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.430516 (+   180us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3688823}
I20250628 07:52:41.433815  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 362) took 3693 ms. Trace:
W20250628 07:52:41.433941   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because the tablet was not found. No further retry: Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:41.435075  2237 rpcz_store.cc:276] 0628 07:52:37.740101 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.741274 (+  1173us) service_pool.cc:225] Handling call
0628 07:52:41.433545 (+3692271us) ts_tablet_manager.cc:1145] Deleting tablet 9579dd44e5e241aea2671be28e16ef7c
0628 07:52:41.433588 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.433753 (+   165us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3691806}
I20250628 07:52:41.437880  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 361) took 3698 ms. Trace:
W20250628 07:52:41.437949   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 9579dd44e5e241aea2671be28e16ef7c because the tablet was not found. No further retry: Not found: Tablet not found: 9579dd44e5e241aea2671be28e16ef7c
I20250628 07:52:41.444103   961 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ae890710f7f44cf899dd19e96fa9181 with delete_type TABLET_DATA_DELETED (replaced by tablet 66353977776545d4a712f58db2aff108) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.440845  1234 tablet_bootstrap.cc:654] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:41.445608   965 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.443293  2237 rpcz_store.cc:276] 0628 07:52:37.739857 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.741817 (+  1960us) service_pool.cc:225] Handling call
0628 07:52:41.437659 (+3695842us) ts_tablet_manager.cc:1145] Deleting tablet b51b51eb951d45a78bd2bffe47834b95
0628 07:52:41.437698 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.437838 (+   140us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3695328}
I20250628 07:52:41.451117  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 368) took 3704 ms. Trace:
I20250628 07:52:41.451488  2237 rpcz_store.cc:276] 0628 07:52:37.746729 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.746820 (+    91us) service_pool.cc:225] Handling call
0628 07:52:41.449887 (+3703067us) ts_tablet_manager.cc:1145] Deleting tablet 83ffa65248fc4db3aecc5f91b648b83f
0628 07:52:41.449934 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.451065 (+  1131us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3702572}
I20250628 07:52:41.453501  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 369) took 3706 ms. Trace:
I20250628 07:52:41.455638  2237 rpcz_store.cc:276] 0628 07:52:37.747031 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.747141 (+   110us) service_pool.cc:225] Handling call
0628 07:52:41.453267 (+3706126us) ts_tablet_manager.cc:1145] Deleting tablet 5ec8c1d9386a4922b55df5e06039858d
0628 07:52:41.453314 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.453446 (+   132us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3705613}
I20250628 07:52:41.456686  1234 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:41.460268  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 367) took 3713 ms. Trace:
I20250628 07:52:41.460669  2237 rpcz_store.cc:276] 0628 07:52:37.746459 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.747320 (+   861us) service_pool.cc:225] Handling call
0628 07:52:41.457259 (+3709939us) ts_tablet_manager.cc:1145] Deleting tablet 5475a89692eb450d82cde521ec8c9a73
0628 07:52:41.457308 (+    49us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.460216 (+  2908us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3709353}
W20250628 07:52:41.462674   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b51b51eb951d45a78bd2bffe47834b95 because the tablet was not found. No further retry: Not found: Tablet not found: b51b51eb951d45a78bd2bffe47834b95
W20250628 07:52:41.463243   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 83ffa65248fc4db3aecc5f91b648b83f because the tablet was not found. No further retry: Not found: Tablet not found: 83ffa65248fc4db3aecc5f91b648b83f
I20250628 07:52:41.463418  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 366) took 3717 ms. Trace:
W20250628 07:52:41.463757   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5ec8c1d9386a4922b55df5e06039858d because the tablet was not found. No further retry: Not found: Tablet not found: 5ec8c1d9386a4922b55df5e06039858d
W20250628 07:52:41.464284   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5475a89692eb450d82cde521ec8c9a73 because the tablet was not found. No further retry: Not found: Tablet not found: 5475a89692eb450d82cde521ec8c9a73
I20250628 07:52:41.413353  2392 raft_consensus.cc:695] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:41.446069   954 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.463945  2237 rpcz_store.cc:276] 0628 07:52:37.746230 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:37.748003 (+  1773us) service_pool.cc:225] Handling call
0628 07:52:41.463186 (+3715183us) ts_tablet_manager.cc:1145] Deleting tablet b2d49e5a06384758bce5e7328b6a10f8
0628 07:52:41.463229 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.463371 (+   142us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3714595}
W20250628 07:52:41.467485   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet b2d49e5a06384758bce5e7328b6a10f8 because the tablet was not found. No further retry: Not found: Tablet not found: b2d49e5a06384758bce5e7328b6a10f8
I20250628 07:52:41.469797  1234 ts_tablet_manager.cc:1397] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.291s	user 0.004s	sys 0.006s
I20250628 07:52:41.472096  1234 raft_consensus.cc:357] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.472736  1234 raft_consensus.cc:383] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:41.473021  1234 raft_consensus.cc:738] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:41.468250  2237 tablet_replica.cc:331] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:41.474249  2237 raft_consensus.cc:2241] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:41.474763  1928 raft_consensus.cc:491] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:41.481328  1928 raft_consensus.cc:513] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.482784  1928 leader_election.cc:290] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:41.473659   965 tablet_service.cc:1515] Processing DeleteTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.460597  1229 raft_consensus.cc:357] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.496614  1229 raft_consensus.cc:383] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:41.496942  1229 raft_consensus.cc:738] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:41.480507  1234 consensus_queue.cc:260] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.505971  1234 ts_tablet_manager.cc:1428] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.036s	user 0.005s	sys 0.000s
I20250628 07:52:41.506640  2237 raft_consensus.cc:2270] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:41.508347  2237 ts_tablet_manager.cc:1905] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:41.476051   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0b251095244548629ae93d52e6c80ccc" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:41.517946  1234 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:41.518095   966 tablet_service.cc:1515] Processing DeleteTablet for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 with delete_type TABLET_DATA_DELETED (replaced by tablet 5febe562643048f083e579c5f7ba0793) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.518905  1229 consensus_queue.cc:260] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.520416  1102 tablet_service.cc:1515] Processing DeleteTablet for tablet 06b9b63649384313948964b3c0dd11cf with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.525794  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:41.526458  1128 raft_consensus.cc:2391] T 7d2a96449ed44d76a015e929d6a6edb0 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:41.531206  1234 tablet_bootstrap.cc:654] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:41.533871   961 tablet_service.cc:1515] Processing DeleteTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 with delete_type TABLET_DATA_DELETED (replaced by tablet 4eac11942d104acab461d483252128c5) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:41.518865  2392 consensus_queue.cc:237] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.537699   965 tablet_service.cc:1515] Processing DeleteTablet for tablet af13b56df157443c994f747e52b0f458 with delete_type TABLET_DATA_DELETED (replaced by tablet f2295f958cb34b658d2dfcf836d25e7f) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:41.538774   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 35fe2e8156cc43d7b3c02092877063b6 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 35fe2e8156cc43d7b3c02092877063b6 already in progress: creating tablet
I20250628 07:52:41.541757   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]). Expected version 2 got 0
I20250628 07:52:41.543745   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:41.523320  1095 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 377) took 1217 ms. Trace:
I20250628 07:52:41.545111  1095 rpcz_store.cc:276] 0628 07:52:40.305870 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.306050 (+   180us) service_pool.cc:225] Handling call
0628 07:52:40.356770 (+ 50720us) spinlock_profiling.cc:243] Waited 8.66 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b420c 00007fba9f0b3dcb 00007fbaa7d2a8f9 00007fbaa6fb2ac0 00007fbaa7095e71 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:40.379445 (+ 22675us) spinlock_profiling.cc:243] Waited 21.9 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b425e 00007fbaa82d8d79 00007fbaa6fb2dc0 00007fbaa7095e8c 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:41.009151 (+629706us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.009235 (+    84us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.046037 (+ 36802us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.302846 (+256809us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.450346 (+147500us) spinlock_profiling.cc:243] Waited 37.6 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b4512 00007fbaa7d527e9 00007fbaa6fb2968 00007fbaa7077083 00007fbaa7072e4f 00007fbaa707360d 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2
0628 07:52:41.522770 (+ 72424us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":156652800}
I20250628 07:52:41.546716  1092 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 384) took 1103 ms. Trace:
I20250628 07:52:41.547036  1092 rpcz_store.cc:276] 0628 07:52:40.442931 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.445226 (+  2295us) service_pool.cc:225] Handling call
0628 07:52:41.038062 (+592836us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.038147 (+    85us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.323895 (+285748us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.522126 (+198231us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.546664 (+ 24538us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:41.516582  2432 leader_election.cc:290] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:41.553488  1181 tablet.cc:2378] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:41.538307   953 tablet_service.cc:1515] Processing DeleteTablet for tablet 06b9b63649384313948964b3c0dd11cf with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:41.555042   758 consensus_peers.cc:489] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:41.545145   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d21243664ea4bbeb4a7c31d8c019486. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:41.559301   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fd67d7d9ba9446c19f1711fb5bd73ebd" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:41.560400   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:41.561419   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:41.563319   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:41.565606   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 586f1b7dd30e4998abebaa1fb2983e2b. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:41.458094   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 115163us
I20250628 07:52:41.585893  2538 raft_consensus.cc:513] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.587617  2538 leader_election.cc:290] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:41.594127  2428 tablet_replica.cc:331] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:41.594849  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:41.595348  1125 raft_consensus.cc:2466] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
W20250628 07:52:41.597226   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 44436.4us
I20250628 07:52:41.598372  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:41.598829  1128 raft_consensus.cc:3058] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:41.599807   757 leader_election.cc:304] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:41.600421  1928 raft_consensus.cc:3058] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:41.607287  1928 raft_consensus.cc:2747] T 0b251095244548629ae93d52e6c80ccc P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:41.614145  1230 tablet_bootstrap.cc:492] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:41.614528  1230 ts_tablet_manager.cc:1397] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.365s	user 0.006s	sys 0.007s
I20250628 07:52:41.621538  2439 consensus_queue.cc:1035] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:41.627275   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b6f2044e4f634f6182beb27ccdc55512"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.628931   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b6f2044e4f634f6182beb27ccdc55512 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.630864   540 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_408f187fe62742609a155742ab328fa6" } new_table_name: "default.Table_60b7b090ea554a1ea11dd815c9b552ee" modify_external_catalogs: true
I20250628 07:52:41.544932   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 2 got 1
I20250628 07:52:41.637315   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:41.638381   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:41.639447   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bfc303c4588d44428984d8203fc05185 (table default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]). Expected version 1 got 0
I20250628 07:52:41.639866  2541 tablet.cc:1722] T bfc303c4588d44428984d8203fc05185 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.640390   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]). Expected version 3 got 0
I20250628 07:52:41.642345  1229 ts_tablet_manager.cc:1428] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.494s	user 0.009s	sys 0.006s
I20250628 07:52:41.646467  1099 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 378) took 1335 ms. Trace:
I20250628 07:52:41.646844  1099 rpcz_store.cc:276] 0628 07:52:40.311052 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.311218 (+   166us) service_pool.cc:225] Handling call
0628 07:52:40.373572 (+ 62354us) spinlock_profiling.cc:243] Waited 3.14 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b420c 00007fba9f0b3dcb 00007fbaa7d2a8f9 00007fbaa6fb2ac0 00007fbaa7095e71 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:40.380548 (+  6976us) spinlock_profiling.cc:243] Waited 3.66 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b425e 00007fbaa82d8d79 00007fbaa6fb2dc0 00007fbaa7095e8c 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:41.009477 (+628929us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.009572 (+    95us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.284417 (+274845us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.519564 (+235147us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.646397 (+126833us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":15661312}
W20250628 07:52:41.657665   920 leader_election.cc:343] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.671573   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_fe7551bb42dd452ab32b051075e3398b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.673311   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_fe7551bb42dd452ab32b051075e3398b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.674059  1128 raft_consensus.cc:2466] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:41.707612  1229 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:41.708432   921 leader_election.cc:304] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:41.709087   538 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_e26652b4c0454211b542434dc5992cd9" } modify_external_catalogs: true
I20250628 07:52:41.709556   538 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_e26652b4c0454211b542434dc5992cd9" } modify_external_catalogs: true
I20250628 07:52:41.718171   986 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "7d2a96449ed44d76a015e929d6a6edb0" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:41.721002  2428 raft_consensus.cc:2241] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: Raft consensus shutting down.
I20250628 07:52:41.738359  1234 tablet_bootstrap.cc:492] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:41.738903  1234 ts_tablet_manager.cc:1397] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.221s	user 0.021s	sys 0.009s
I20250628 07:52:41.744093   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]). Expected version 1 got 0
W20250628 07:52:41.745280   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 86877.2us
I20250628 07:52:41.745404   546 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]). Expected version 1 got 0
I20250628 07:52:41.747090  2567 tablet.cc:1722] T bfc303c4588d44428984d8203fc05185 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.748118  2569 tablet.cc:1722] T 46e0aa090f544e8aa0a426c61925f345 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.718489  1230 raft_consensus.cc:357] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.759920  1230 raft_consensus.cc:383] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:41.760205  1230 raft_consensus.cc:738] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:41.762362  1098 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 380) took 1435 ms. Trace:
I20250628 07:52:41.762758  1098 rpcz_store.cc:276] 0628 07:52:40.327016 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.327210 (+   194us) service_pool.cc:225] Handling call
0628 07:52:40.357153 (+ 29943us) spinlock_profiling.cc:243] Waited 1.24 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b420c 00007fba9f0b3dcb 00007fbaa7d2a8f9 00007fbaa6fb2ac0 00007fbaa7095e71 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:40.442195 (+ 85042us) spinlock_profiling.cc:243] Waited 24.9 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b3ee1 00007fba9f0b425e 00007fbaa82d8d79 00007fbaa6fb2dc0 00007fbaa7095e8c 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:41.029422 (+587227us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.029531 (+   109us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.347673 (+318142us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.703473 (+355800us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.762306 (+ 58833us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":60198400}
I20250628 07:52:41.763402  1094 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:54948 (request call id 375) took 1467 ms. Trace:
I20250628 07:52:41.766117   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]) successfully deleted
W20250628 07:52:41.766723   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because the tablet was not found. No further retry: Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:41.767740  2428 pending_rounds.cc:70] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Trying to abort 1 pending ops.
W20250628 07:52:41.769635  1062 leader_election.cc:343] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.770076  1062 leader_election.cc:304] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:41.770768  2410 raft_consensus.cc:2747] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:41.771063  1062 leader_election.cc:343] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.771436  1062 leader_election.cc:304] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:41.772038  2410 raft_consensus.cc:2747] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:41.772325  1062 leader_election.cc:343] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.781224  2111 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" instance_seqno: 1751097130210398) as {username='slave'} at 127.0.107.131:44187; Asking this server to re-register.
I20250628 07:52:41.772709  1062 leader_election.cc:304] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:41.763814  1094 rpcz_store.cc:276] 0628 07:52:40.296085 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.296265 (+   180us) service_pool.cc:225] Handling call
0628 07:52:40.451819 (+155554us) spinlock_profiling.cc:243] Waited 35 ms on lock 0x7b5800000098. stack: 00007fba9f0bf4b3 00007fba9f0bf3b0 00007fba9ec3b574 00000000004fa174 00000000004fa109 00007fba9ef349b1 00007fba9f0b420c 00007fba9f0b3dcb 00007fbaa7d2a8f9 00007fbaa6fb2ac0 00007fbaa7095e71 00007fbaa7073203 00007fbaa7023c99 00007fbaa56f0d34 00007fbaa56f0cc2 00007fbaa56f0bf1
0628 07:52:41.038282 (+586463us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.042787 (+  4505us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.398725 (+355938us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.762939 (+364214us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.763351 (+   412us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":80559488}
I20250628 07:52:41.782171  2410 raft_consensus.cc:2747] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:41.783078  2543 raft_consensus.cc:2802] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:41.778043  2514 ts_tablet_manager.cc:1918] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:41.783533  2543 raft_consensus.cc:491] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:41.783702  2514 log.cc:1199] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/d87dd96d00964e12acd6e380a62c94d8
I20250628 07:52:41.783835  2543 raft_consensus.cc:3058] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:41.788725   958 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:35848 (request call id 408) took 1057 ms. Trace:
I20250628 07:52:41.789042   958 rpcz_store.cc:276] 0628 07:52:40.731446 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.828060 (+ 96614us) service_pool.cc:225] Handling call
0628 07:52:41.504395 (+676335us) ts_tablet_manager.cc:663] Acquired tablet manager lock
0628 07:52:41.514063 (+  9668us) ts_tablet_manager.cc:676] Creating new metadata...
0628 07:52:41.582181 (+ 68118us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.760437 (+178256us) tablet_replica.cc:206] Creating consensus instance
0628 07:52:41.788670 (+ 28233us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:41.790139   547 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 22b47b39f8eb45a0b5d2c5348bfbc55c
I20250628 07:52:41.790181   545 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_59e9b18b772e4d5dafd78313d4b47800 [id=9a2c30b2d6a6409dbd3565a4b74ba210] alter complete (version 1)
I20250628 07:52:41.791518   547 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 433) took 2218 ms. Trace:
W20250628 07:52:41.793444   540 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5cacde13bc29458da81f29fb4fa2945a consensus state has no leader
I20250628 07:52:41.798406  1230 consensus_queue.cc:260] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:41.790035   539 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b] alter complete (version 3)
I20250628 07:52:41.804167   792 tablet_service.cc:1515] Processing DeleteTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:40 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.767999  2428 pending_rounds.cc:77] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Aborting op as it isn't in flight: id { term: 2 index: 1 } timestamp: 7172493973658632192 op_type: NO_OP noop_request { }
I20250628 07:52:41.805159  2428 raft_consensus.cc:2887] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:52:41.805483  2428 raft_consensus.cc:2270] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:41.806273   757 consensus_peers.cc:489] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d87dd96d00964e12acd6e380a62c94d8. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:41.791813   547 rpcz_store.cc:276] 0628 07:52:39.572697 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.923882 (+1351185us) service_pool.cc:225] Handling call
0628 07:52:40.925173 (+  1291us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:40.925865 (+   692us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:40.926012 (+   147us) catalog_manager.cc:2799] Locking tablets
0628 07:52:40.926479 (+   467us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:40.967313 (+ 40834us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:40.967489 (+   176us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:41.790049 (+822560us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:41.791449 (+  1400us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.928388 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.928860 (+   472us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.928886 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:40.928900 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:40.929249 (+   349us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.929293 (+    44us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.929314 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.929586 (+   272us) tablet.cc:699] Row locks acquired
0628 07:52:40.929603 (+    17us) write_op.cc:260] PREPARE: finished
0628 07:52:40.929722 (+   119us) write_op.cc:270] Start()
0628 07:52:40.929822 (+   100us) write_op.cc:276] Timestamp: P: 1751097160929656 usec, L: 0
0628 07:52:40.929843 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.932576 (+  2733us) log.cc:844] Serialized 771 byte log entry
0628 07:52:40.933243 (+   667us) spinlock_profiling.cc:243] Waited 2.28 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:40.965280 (+ 32037us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.965547 (+   267us) write_op.cc:301] APPLY: starting
0628 07:52:40.965595 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.965686 (+    91us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.965702 (+    16us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.965983 (+   281us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.966095 (+   112us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:40.966129 (+    34us) write_op.cc:312] APPLY: finished
0628 07:52:40.966269 (+   140us) log.cc:844] Serialized 29 byte log entry
0628 07:52:40.966382 (+   113us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.966440 (+    58us) write_op.cc:454] Released schema lock
0628 07:52:40.966467 (+    27us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:40.966651 (+   184us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.966674 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":144,"apply.run_cpu_time_us":1074,"apply.run_wall_time_us":1073,"num_ops":2,"prepare.queue_time_us":99,"prepare.run_cpu_time_us":2812,"prepare.run_wall_time_us":4950,"replication_time_us":35336,"spinlock_wait_cycles":5234944}]]}
I20250628 07:52:41.808844  1094 tablet_service.cc:1515] Processing DeleteTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:40 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:41.809322   757 consensus_peers.cc:489] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:41.809909   757 leader_election.cc:304] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:41.810431  1176 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:41.810580  2439 raft_consensus.cc:3058] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:41.810348  2566 tablet.cc:1722] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.812368  2514 ts_tablet_manager.cc:1939] T d87dd96d00964e12acd6e380a62c94d8 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:41.815105  2569 tablet.cc:1722] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.818715  2439 raft_consensus.cc:2747] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:41.804364  2571 tablet.cc:1722] T 46e0aa090f544e8aa0a426c61925f345 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:41.824345  2512 catalog_manager.cc:1261] Loaded cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:41.830247   792 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.833595   789 tablet_service.cc:1515] Processing DeleteTablet for tablet e15d063a4bca40d480cba08733ec1ec3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.835223   789 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.836447   792 tablet_service.cc:1515] Processing DeleteTablet for tablet 8e3e0e52b4924e67904d3acfa4dd5752 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:35 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:41.837651   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_061d0f52cd51486a88f1e59480dffc7f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.839644   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_061d0f52cd51486a88f1e59480dffc7f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.844673   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_992ef1fbcdb840c382cc60a218166855"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.844565   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5cacde13bc29458da81f29fb4fa2945a consensus state has no leader
W20250628 07:52:41.846217   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_992ef1fbcdb840c382cc60a218166855 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.849625  1094 tablet_service.cc:1515] Processing DeleteTablet for tablet 63115b7be4a94b639c18601d438ed947 with delete_type TABLET_DATA_DELETED (replaced by tablet f0d48652a4da42f3b26b42f65c8db51b) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.850139  1098 tablet_service.cc:1515] Processing DeleteTablet for tablet f845f32699044867af6f0f2b655cad8e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.850462  1090 tablet_service.cc:1515] Processing DeleteTablet for tablet 5fa4299a9e0f4b409d04c8c5900901c0 with delete_type TABLET_DATA_DELETED (replaced by tablet 0bdb4c499be441379663830f26d80204) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.863608  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 391) took 1212 ms. Trace:
I20250628 07:52:41.863972  2514 rpcz_store.cc:276] 0628 07:52:40.651368 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.651557 (+   189us) service_pool.cc:225] Handling call
0628 07:52:40.701970 (+ 50413us) ts_tablet_manager.cc:1145] Deleting tablet d87dd96d00964e12acd6e380a62c94d8
0628 07:52:41.077229 (+375259us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.608722 (+531493us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.777813 (+169091us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.863529 (+ 85716us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":273317,"tablet-delete.queue_time_us":49562,"thread_start_us":74781,"threads_started":1}
I20250628 07:52:41.867038  2514 tablet_replica.cc:331] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:41.867632   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]) successfully deleted
I20250628 07:52:41.874161   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_ecc8fdb9f92341d38931d4b776b8114a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:41.875640  1230 ts_tablet_manager.cc:1428] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.261s	user 0.005s	sys 0.002s
W20250628 07:52:41.875605   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ecc8fdb9f92341d38931d4b776b8114a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.877388  1230 tablet_bootstrap.cc:492] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:41.884557  2514 raft_consensus.cc:2241] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:41.885129  2514 raft_consensus.cc:2270] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:41.887821  2514 ts_tablet_manager.cc:1905] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:41.895565  1036 tablet.cc:2378] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:41.720544  2402 consensus_queue.cc:1035] T 86591ef37c8547189ff8ef270e9bfa9a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.084s
I20250628 07:52:41.901499  1091 tablet_service.cc:1515] Processing DeleteTablet for tablet e15d063a4bca40d480cba08733ec1ec3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:34 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.914829  1229 tablet_bootstrap.cc:654] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:41.915060   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_b63e7c6a006d4e12b58a66a5ef9615b4"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:41.916591   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:41.944129  2514 ts_tablet_manager.cc:1918] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:41.945488  2541 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:41482 (request call id 405) took 1256 ms. Trace:
W20250628 07:52:41.946614  2558 consensus_queue.cc:1162] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "82f43cdfab544b17a6a71033983e65d4" status { last_received { term: 2 index: 1 } last_committed_idx: 0 last_received_current_leader { term: 2 index: 1 } }
I20250628 07:52:41.874193  2103 ts_manager.cc:194] Registered new tserver with Master: 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:41.876852  2219 debug-util.cc:398] Leaking SignalData structure 0x7b080007c300 after lost signal to thread 1440
W20250628 07:52:41.957679  2219 debug-util.cc:398] Leaking SignalData structure 0x7b080007c400 after lost signal to thread 2228
W20250628 07:52:41.958123  2219 debug-util.cc:398] Leaking SignalData structure 0x7b080006e600 after lost signal to thread 2470
W20250628 07:52:41.963719   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 67) took 2571 ms (client timeout 1000 ms). Trace:
I20250628 07:52:41.981307  2512 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: loading cluster ID for follower catalog manager: success
I20250628 07:52:41.993384   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312] alter complete (version 1)
W20250628 07:52:42.010133   542 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 74) took 3720 ms (client timeout 1000 ms). Trace:
W20250628 07:52:42.005878   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 217265us
W20250628 07:52:42.031857   920 leader_election.cc:343] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:41.970145  1090 tablet_service.cc:1515] Processing DeleteTablet for tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 with delete_type TABLET_DATA_DELETED (replaced by tablet 7986ee140247478eafc2d95aaec8f8b4) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:41.971868  1091 tablet_service.cc:1515] Processing DeleteTablet for tablet fa3ec14241e344029df3e39473181078 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:30 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.010396  1094 tablet_service.cc:1515] Processing DeleteTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 with delete_type TABLET_DATA_DELETED (replaced by tablet 80ee935a2d58422fb1bfbc9103940920) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.013247  1098 tablet_service.cc:1515] Processing DeleteTablet for tablet d87dd96d00964e12acd6e380a62c94d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.021982  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.034788  2514 log.cc:1199] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/c4fcefaa9e954b52bbbee6e0a2f2f939
I20250628 07:52:42.035672  2514 ts_tablet_manager.cc:1939] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:42.024834  1230 tablet_bootstrap.cc:654] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:42.051510   542 rpcz_store.cc:269] 0628 07:52:38.289448 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.318448 (+ 29000us) service_pool.cc:225] Handling call
0628 07:52:42.009967 (+3691519us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":95170,"reported_tablets":42}
W20250628 07:52:41.964020   545 rpcz_store.cc:269] 0628 07:52:39.391924 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.414909 (+ 22985us) service_pool.cc:225] Handling call
0628 07:52:40.413930 (+999021us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.413958 (+    28us) write_op.cc:454] Released schema lock
0628 07:52:41.963643 (+1549685us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.390847 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.391293 (+   446us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.391322 (+    29us) write_op.cc:435] Acquired schema lock
0628 07:52:40.391337 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:40.391697 (+   360us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.391744 (+    47us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.391781 (+    37us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:40.392093 (+   312us) tablet.cc:699] Row locks acquired
0628 07:52:40.392108 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:40.392221 (+   113us) write_op.cc:270] Start()
0628 07:52:40.392306 (+    85us) write_op.cc:276] Timestamp: P: 1751097160392170 usec, L: 0
0628 07:52:40.392326 (+    20us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.392865 (+   539us) log.cc:844] Serialized 913 byte log entry
0628 07:52:40.400553 (+  7688us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.400912 (+   359us) write_op.cc:301] APPLY: starting
0628 07:52:40.400963 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.401061 (+    98us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.401161 (+   100us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.401484 (+   323us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.401615 (+   131us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:40.401661 (+    46us) write_op.cc:312] APPLY: finished
0628 07:52:40.401843 (+   182us) log.cc:844] Serialized 29 byte log entry
0628 07:52:40.401990 (+   147us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.402059 (+    69us) write_op.cc:454] Released schema lock
0628 07:52:40.402088 (+    29us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:40.693496 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.693958 (+   462us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.693984 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:40.693996 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:40.694275 (+   279us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.694318 (+    43us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.694342 (+    24us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:40.694495 (+   153us) tablet.cc:699] Row locks acquired
0628 07:52:40.694507 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:40.694638 (+   131us) write_op.cc:270] Start()
0628 07:52:40.694733 (+    95us) write_op.cc:276] Timestamp: P: 1751097160694553 usec, L: 0
0628 07:52:40.694749 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.695252 (+   503us) log.cc:844] Serialized 356 byte log entry
0628 07:52:40.896691 (+201439us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.897003 (+   312us) write_op.cc:301] APPLY: starting
0628 07:52:40.897061 (+    58us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.897192 (+   131us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.897272 (+    80us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.897482 (+   210us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.897623 (+   141us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:40.897664 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:40.897835 (+   171us) log.cc:844] Serialized 23 byte log entry
0628 07:52:40.897966 (+   131us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.898017 (+    51us) write_op.cc:454] Released schema lock
0628 07:52:40.898048 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:40.898259 (+   211us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.898280 (+    21us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":37,"child_traces":[["op",{"apply.queue_time_us":214,"apply.run_cpu_time_us":1354,"apply.run_wall_time_us":1354,"num_ops":2,"prepare.queue_time_us":850,"prepare.run_cpu_time_us":2334,"prepare.run_wall_time_us":2336,"raft.queue_time_us":177,"raft.run_cpu_time_us":670,"raft.run_wall_time_us":670,"replication_time_us":8119}],["op",{"apply.queue_time_us":117,"apply.run_cpu_time_us":1240,"apply.run_wall_time_us":1240,"num_ops":1,"prepare.queue_time_us":133,"prepare.run_cpu_time_us":2054,"prepare.run_wall_time_us":2054,"raft.queue_time_us":214,"raft.run_cpu_time_us":626,"raft.run_wall_time_us":626,"replication_time_us":201795}]]}
I20250628 07:52:42.053648  2541 rpcz_store.cc:276] 0628 07:52:40.688991 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.689158 (+   167us) service_pool.cc:225] Handling call
0628 07:52:41.945418 (+1256260us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.690194 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
0628 07:52:40.690615 (+   421us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
0628 07:52:40.690641 (+    26us) alter_schema_op.cc:66] Acquired schema lock
0628 07:52:40.690707 (+    66us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
0628 07:52:40.690833 (+   126us) alter_schema_op.cc:128] START. Timestamp: 1751097160690717
0628 07:52:40.690858 (+    25us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.691388 (+   530us) log.cc:844] Serialized 173 byte log entry
0628 07:52:41.274811 (+583423us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.594072 (+319261us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
0628 07:52:41.641525 (+ 47453us) log.cc:844] Serialized 5 byte log entry
0628 07:52:41.939893 (+298368us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:41.940895 (+  1002us) log.cc:844] Serialized 14 byte log entry
0628 07:52:41.945270 (+  4375us) alter_schema_op.cc:73] Released schema lock
0628 07:52:41.945302 (+    32us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
Metrics: {"child_traces":[["op",{"apply.queue_time_us":319065,"prepare.queue_time_us":147,"prepare.run_cpu_time_us":9826,"prepare.run_wall_time_us":60070,"raft.queue_time_us":1437,"raft.run_cpu_time_us":1516,"raft.run_wall_time_us":1836,"replication_time_us":583837,"thread_start_us":443043,"threads_started":2,"wal-append.queue_time_us":96189}]]}
I20250628 07:52:42.081825   540 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9287e3b4d3614c488fb35212433f337f" } modify_external_catalogs: true
I20250628 07:52:42.082515   540 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_9287e3b4d3614c488fb35212433f337f" } modify_external_catalogs: true
I20250628 07:52:42.085614   541 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_958abb2126a34f57af7942df2b6bbb9a" } new_table_name: "default.Table_9bd0a6cbc9e8466cb54f871dc9938e95" modify_external_catalogs: true
I20250628 07:52:42.090008   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_402049f057084c25a8b2a78d7fbdc2a0" } new_table_name: "default.Table_c8df75abf8c64fa4adc3102bb5f22ff2" modify_external_catalogs: true
I20250628 07:52:42.040561  2237 ts_tablet_manager.cc:1918] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:42.092995  2237 log.cc:1199] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:42.094445  2237 ts_tablet_manager.cc:1939] T 9ba2a10acdf0462bb05628351cdbbbe4 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:42.095224  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 370) took 3823 ms. Trace:
I20250628 07:52:42.095584  2237 rpcz_store.cc:276] 0628 07:52:38.272071 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.274333 (+  2262us) service_pool.cc:225] Handling call
0628 07:52:41.468126 (+3193793us) ts_tablet_manager.cc:1145] Deleting tablet 9ba2a10acdf0462bb05628351cdbbbe4
0628 07:52:41.468158 (+    32us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:41.857581 (+389423us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.040350 (+182769us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.095171 (+ 54821us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3193335}
I20250628 07:52:42.096777   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 9ba2a10acdf0462bb05628351cdbbbe4 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]) successfully deleted
I20250628 07:52:42.106038  2541 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:42.115720   544 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a] alter complete (version 2)
I20250628 07:52:42.058967  1234 raft_consensus.cc:357] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.116417  1234 raft_consensus.cc:383] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.116698  1234 raft_consensus.cc:738] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.117537  1234 consensus_queue.cc:260] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:42.119211   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5febe562643048f083e579c5f7ba0793 consensus state has no leader
I20250628 07:52:42.121585  1234 ts_tablet_manager.cc:1428] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.382s	user 0.008s	sys 0.000s
I20250628 07:52:42.145183   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]) successfully deleted
I20250628 07:52:42.151849  1230 tablet_bootstrap.cc:492] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:42.152321  1230 ts_tablet_manager.cc:1397] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.275s	user 0.005s	sys 0.004s
I20250628 07:52:42.155010  1230 raft_consensus.cc:357] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.158068  1230 raft_consensus.cc:383] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.158310  1230 raft_consensus.cc:738] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.158944  1230 consensus_queue.cc:260] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.164320  2566 tablet.cc:1722] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:42.176854  2567 tablet.cc:1722] T fceb8ef4ddb44e7eb11be71a5087bd27 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:42.181267   538 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet fd67d7d9ba9446c19f1711fb5bd73ebd
I20250628 07:52:42.181489  1230 ts_tablet_manager.cc:1428] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.029s	user 0.007s	sys 0.001s
I20250628 07:52:42.184948  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:54948
W20250628 07:52:42.186868   541 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet b759dcf28b784ab99b7aed5d22c651dc consensus state has no leader
I20250628 07:52:42.096966  2237 tablet_replica.cc:331] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:42.191926  2237 raft_consensus.cc:2241] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:42.199035   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8579b9af0fe6471c99b0ab6cebc7ed29"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:42.201233   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.203907   541 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4107fb91f5704f38b337a443ee54704c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:42.205806   541 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4107fb91f5704f38b337a443ee54704c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.208215   542 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3cf97e46be3049c0b1faa4e17cfa70eb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:42.149673  2546 consensus_queue.cc:1035] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.009s
I20250628 07:52:42.180311   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_af46d3e348f94f0fbd27c3f97a2028ff"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:42.215291  2428 ts_tablet_manager.cc:1905] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:42.215653   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_af46d3e348f94f0fbd27c3f97a2028ff in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:42.221668   542 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3cf97e46be3049c0b1faa4e17cfa70eb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.210995   792 tablet_service.cc:1515] Processing DeleteTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:42.238615  2237 raft_consensus.cc:2270] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:42.240681  2237 ts_tablet_manager.cc:1905] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:42.247184   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet c3d1bc8483374a2a917451a43b3d42ed
I20250628 07:52:42.249323  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet c3d1bc8483374a2a917451a43b3d42ed with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.252115   792 tablet_service.cc:1515] Processing DeleteTablet for tablet c3d1bc8483374a2a917451a43b3d42ed with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:42.265553  2410 raft_consensus.cc:491] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.266028  2410 raft_consensus.cc:513] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.198431  2543 raft_consensus.cc:513] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.268458  2410 leader_election.cc:290] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:42.273586   921 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.131:37709, user_credentials={real_user=slave}} blocked reactor thread for 500289us
I20250628 07:52:42.274298   921 leader_election.cc:304] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:42.275836  2411 raft_consensus.cc:2802] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:42.276388  2411 raft_consensus.cc:695] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
W20250628 07:52:42.283563   984 thread.cc:640] apply [worker] (thread pool) Time spent creating pthread: real 0.562s	user 0.021s	sys 0.031s
W20250628 07:52:42.283740   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 60) took 2372 ms (client timeout 1000 ms). Trace:
W20250628 07:52:42.283890   984 thread.cc:606] apply [worker] (thread pool) Time spent starting thread: real 0.564s	user 0.021s	sys 0.031s
I20250628 07:52:42.290393  2543 leader_election.cc:290] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.291106  2411 consensus_queue.cc:237] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.308188   540 master_service.cc:948] ReplaceTablet: received request to replace tablet 88d4a84c8ed7475c9bf043aaba8ad624 from {username='slave'} at 127.0.0.1:47964
W20250628 07:52:42.283535   981 thread.cc:640] wal-append [worker] (thread pool) Time spent creating pthread: real 0.761s	user 0.005s	sys 0.014s
I20250628 07:52:42.227042  1234 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:42.326692  1230 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:42.330261  2512 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: acquiring CA information for follower catalog manager: success
I20250628 07:52:42.326757  2428 ts_tablet_manager.cc:1918] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 2.1
I20250628 07:52:42.335546  1230 tablet_bootstrap.cc:654] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:42.335716  2428 log.cc:1199] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:42.336400  2428 ts_tablet_manager.cc:1939] T 2ae890710f7f44cf899dd19e96fa9181 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:42.337327  2514 tablet_replica.cc:331] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:42.338506  2514 raft_consensus.cc:2241] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:42.339167  2514 raft_consensus.cc:2270] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:42.313498   981 thread.cc:606] wal-append [worker] (thread pool) Time spent starting thread: real 0.792s	user 0.005s	sys 0.014s
I20250628 07:52:42.340975   981 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.107.129:48103 (request call id 369) took 1389 ms. Trace:
I20250628 07:52:42.341305   981 rpcz_store.cc:276] 0628 07:52:40.951435 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.958743 (+  7308us) service_pool.cc:225] Handling call
0628 07:52:41.265059 (+306316us) spinlock_profiling.cc:243] Waited 1.17 ms on lock 0x7b5800000098. stack: 00007ff6ffd094b3 00007ff6ffd093b0 00007ff6ff885574 00000000004fa174 00000000004fa109 00007ff6ffb7e9b1 00007ff6ffcfe20c 00007ff6ffcfddcb 00007ff7089748f9 00007ff707bfcac0 00007ff707cc37bb 00007ff707cc3841 00007ff707c86860 00007ff707c71c29 00007ff707c716d2 00007ff706419744
0628 07:52:41.503855 (+238796us) spinlock_profiling.cc:243] Waited 2.51 ms on lock 0x7b5800000098. stack: 00007ff6ffd094b3 00007ff6ffd093b0 00007ff6ff885574 00000000004fa174 00000000004fa109 00007ff6ffb7e9b1 00007ff6ffcfdee1 00007ff6ffcfe25e 00007ff708f22d79 00007ff707bfcdc0 00007ff707cc37d4 00007ff707cc3841 00007ff707c86860 00007ff707c71c29 00007ff707c716d2 00007ff706419744
0628 07:52:41.504000 (+   145us) raft_consensus.cc:1469] Updating replica for 1 ops
0628 07:52:41.504321 (+   321us) raft_consensus.cc:1532] Early marking committed up to index 0
0628 07:52:41.504349 (+    28us) raft_consensus.cc:1537] Triggering prepare for 1 ops
0628 07:52:41.520451 (+ 16102us) log.cc:844] Serialized 26 byte log entry
0628 07:52:42.340587 (+820136us) raft_consensus.cc:1643] Marking committed up to 0
0628 07:52:42.340629 (+    42us) raft_consensus.cc:1695] Filling consensus response to leader.
0628 07:52:42.340706 (+    77us) raft_consensus.cc:1666] Waiting on the replicates to finish logging
0628 07:52:42.340755 (+    49us) raft_consensus.cc:1683] finished
0628 07:52:42.340764 (+     9us) raft_consensus.cc:1689] UpdateReplicas() finished
0628 07:52:42.340883 (+   119us) inbound_call.cc:173] Queueing success response
Metrics: {"raft.queue_time_us":243,"raft.run_cpu_time_us":180,"raft.run_wall_time_us":2677,"spinlock_wait_cycles":8456832,"thread_start_us":819829,"threads_started":1,"wal-append.queue_time_us":710164}
I20250628 07:52:42.347232  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:42.347769  1125 raft_consensus.cc:3058] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:42.347900  2551 thread.cc:640] wal-append [worker] (thread pool) Time spent creating pthread: real 0.596s	user 0.035s	sys 0.074s
W20250628 07:52:42.348228  2551 thread.cc:606] wal-append [worker] (thread pool) Time spent starting thread: real 0.781s	user 0.035s	sys 0.074s
I20250628 07:52:42.355492  1125 raft_consensus.cc:2466] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:42.356506  2567 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
W20250628 07:52:42.357455  2526 thread.cc:640] prepare [worker] (thread pool) Time spent creating pthread: real 0.742s	user 0.004s	sys 0.024s
W20250628 07:52:42.357786  2526 thread.cc:606] prepare [worker] (thread pool) Time spent starting thread: real 0.946s	user 0.005s	sys 0.024s
W20250628 07:52:42.360054   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 68) took 1706 ms (client timeout 1000 ms). Trace:
I20250628 07:52:42.364336   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8226c926201040c993804d2ae8450202" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:42.364192  2536 consensus_queue.cc:1035] T 7d2a96449ed44d76a015e929d6a6edb0 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:42.364883   824 raft_consensus.cc:3058] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.373293   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "77234b63748849b6996c7aa0ca23ef05" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:42.384644   921 leader_election.cc:304] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:42.384761  1229 tablet_bootstrap.cc:492] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:42.385249  1229 ts_tablet_manager.cc:1397] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.678s	user 0.015s	sys 0.011s
I20250628 07:52:42.387440   824 raft_consensus.cc:2466] T 8226c926201040c993804d2ae8450202 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:42.390102  2585 tablet.cc:1722] T 8feee3be1a664cae9da417b190163a94 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 3
W20250628 07:52:42.397841   922 thread.cc:640] raft [worker] (thread pool) Time spent creating pthread: real 0.525s	user 0.014s	sys 0.005s
W20250628 07:52:42.398149   922 thread.cc:606] raft [worker] (thread pool) Time spent starting thread: real 0.610s	user 0.014s	sys 0.005s
I20250628 07:52:42.400614  1234 tablet_bootstrap.cc:654] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:42.401832  2546 raft_consensus.cc:2802] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:42.402295  2546 raft_consensus.cc:695] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:42.402539  1229 raft_consensus.cc:357] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.403170  1229 raft_consensus.cc:383] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.403415  1229 raft_consensus.cc:738] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.403030  2546 consensus_queue.cc:237] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.403936  1229 consensus_queue.cc:260] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.406049  1229 ts_tablet_manager.cc:1428] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.021s	user 0.004s	sys 0.001s
I20250628 07:52:42.421706  2428 tablet_replica.cc:331] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:42.422562  1229 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:42.372099  2514 ts_tablet_manager.cc:1905] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:42.374765  1062 leader_election.cc:343] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:42.426193  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 51474.4us
I20250628 07:52:42.426827  2410 raft_consensus.cc:491] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.427194  2410 raft_consensus.cc:513] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.428810  2410 leader_election.cc:290] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:42.430828  2568 tablet.cc:1722] T ef5d1e5d358e424d818afeeebab3cfa2 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:42.432336  1229 tablet_bootstrap.cc:654] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:42.435967  2428 raft_consensus.cc:2241] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:42.392030  1230 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:42.416702  2589 tablet.cc:1722] T bed1ea07e3d44c08adf37ced9dbef85e P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:42.436527  1230 ts_tablet_manager.cc:1397] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.110s	user 0.006s	sys 0.007s
I20250628 07:52:42.442391   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:42.445112  2428 raft_consensus.cc:2270] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:42.450398  1062 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.462293  2541 tablet.cc:1722] T bed1ea07e3d44c08adf37ced9dbef85e P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:42.472239   541 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0979356e8b1a441fa942c0359545cfc9" } modify_external_catalogs: true
I20250628 07:52:42.472797   541 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_0979356e8b1a441fa942c0359545cfc9" } modify_external_catalogs: true
I20250628 07:52:42.475617  1234 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:42.476078  1234 ts_tablet_manager.cc:1397] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.249s	user 0.011s	sys 0.001s
I20250628 07:52:42.477075   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_786e13bb34124bc0bb4f5dae8326f725" } new_table_name: "default.Table_2780965e7e5d4d528b1d63995873222f" modify_external_catalogs: true
I20250628 07:52:42.481753  2441 raft_consensus.cc:491] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.482148  2441 raft_consensus.cc:513] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.483882  2441 leader_election.cc:290] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.484745  1234 raft_consensus.cc:357] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.487493  1234 raft_consensus.cc:383] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.488781  1234 raft_consensus.cc:738] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.489279   792 tablet_service.cc:1515] Processing DeleteTablet for tablet 9ba2a10acdf0462bb05628351cdbbbe4 with delete_type TABLET_DATA_DELETED (replaced by tablet 80ee935a2d58422fb1bfbc9103940920) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:42.490428  1234 consensus_queue.cc:260] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.491473   789 tablet_service.cc:1515] Processing DeleteTablet for tablet f845f32699044867af6f0f2b655cad8e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:26 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:42.497520  1234 ts_tablet_manager.cc:1428] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.021s	user 0.005s	sys 0.000s
I20250628 07:52:42.500061  1234 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:42.502391  2144 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:42384 (request call id 0) took 7991 ms. Trace:
I20250628 07:52:42.508550  2441 raft_consensus.cc:491] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.508945  2441 raft_consensus.cc:513] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.510800  2441 leader_election.cc:290] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.518872  1230 raft_consensus.cc:357] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.519532  1230 raft_consensus.cc:383] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.519785  1230 raft_consensus.cc:738] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.523598  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet 29a77cd9dc244d239a518e004c49e3c8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:25 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.524200  1091 tablet_service.cc:1515] Processing DeleteTablet for tablet a8518c82c188484c8554d3115637f5be with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.524775  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet a447edf0960c4739930450a8c24ff6f7 with delete_type TABLET_DATA_DELETED (replaced by tablet 924b921830b7403596ebcc6cf80dc24d) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.531800  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:42.532312  1126 raft_consensus.cc:2466] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
W20250628 07:52:42.360373   544 rpcz_store.cc:269] 0628 07:52:40.653896 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.706690 (+ 52794us) service_pool.cc:225] Handling call
0628 07:52:42.359987 (+1653297us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:42.033002 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:42.033495 (+   493us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:42.033524 (+    29us) write_op.cc:435] Acquired schema lock
0628 07:52:42.033537 (+    13us) tablet.cc:660] Decoding operations
0628 07:52:42.033836 (+   299us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:42.033883 (+    47us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:42.033908 (+    25us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:42.034065 (+   157us) tablet.cc:699] Row locks acquired
0628 07:52:42.034078 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:42.034195 (+   117us) write_op.cc:270] Start()
0628 07:52:42.034884 (+   689us) write_op.cc:276] Timestamp: P: 1751097162034128 usec, L: 0
0628 07:52:42.034905 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:42.060036 (+ 25131us) log.cc:844] Serialized 356 byte log entry
0628 07:52:42.089101 (+ 29065us) spinlock_profiling.cc:243] Waited 23.4 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:42.106271 (+ 17170us) op_driver.cc:464] REPLICATION: finished
0628 07:52:42.114427 (+  8156us) write_op.cc:301] APPLY: starting
0628 07:52:42.114479 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:42.114560 (+    81us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:42.114594 (+    34us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:42.114748 (+   154us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:42.114871 (+   123us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:42.114911 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:42.115072 (+   161us) log.cc:844] Serialized 23 byte log entry
0628 07:52:42.115210 (+   138us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:42.115261 (+    51us) write_op.cc:454] Released schema lock
0628 07:52:42.115293 (+    32us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:42.115486 (+   193us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:42.115509 (+    23us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":97892,"reported_tablets":38,"child_traces":[["op",{"apply.queue_time_us":8023,"apply.run_cpu_time_us":1035,"apply.run_wall_time_us":1035,"num_ops":1,"prepare.queue_time_us":2167,"prepare.run_cpu_time_us":4138,"prepare.run_wall_time_us":56349,"raft.queue_time_us":7590,"raft.run_cpu_time_us":634,"raft.run_wall_time_us":635,"replication_time_us":71279,"spinlock_wait_cycles":53898624}]]}
I20250628 07:52:42.534852   544 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_848608eca1b140e5bb004562ebd3fde8" } new_table_name: "default.Table_6fd35d7894eb4e9891f539a62d743e62" modify_external_catalogs: true
W20250628 07:52:42.284044   543 rpcz_store.cc:269] 0628 07:52:39.911883 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.658566 (+746683us) service_pool.cc:225] Handling call
0628 07:52:42.283514 (+1624948us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:41.095396 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:41.095801 (+   405us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:41.095826 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:41.095840 (+    14us) tablet.cc:660] Decoding operations
0628 07:52:41.096080 (+   240us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:41.096119 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:41.096142 (+    23us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:41.096274 (+   132us) tablet.cc:699] Row locks acquired
0628 07:52:41.096285 (+    11us) write_op.cc:260] PREPARE: finished
0628 07:52:41.096387 (+   102us) write_op.cc:270] Start()
0628 07:52:41.096468 (+    81us) write_op.cc:276] Timestamp: P: 1751097161096322 usec, L: 0
0628 07:52:41.096482 (+    14us) op_driver.cc:348] REPLICATION: starting
0628 07:52:41.135675 (+ 39193us) log.cc:844] Serialized 533 byte log entry
0628 07:52:41.137472 (+  1797us) spinlock_profiling.cc:243] Waited 38.7 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:41.181778 (+ 44306us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.182008 (+   230us) write_op.cc:301] APPLY: starting
0628 07:52:41.182057 (+    49us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:41.182135 (+    78us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:41.182153 (+    18us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:41.182323 (+   170us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:41.182440 (+   117us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:41.182479 (+    39us) write_op.cc:312] APPLY: finished
0628 07:52:41.183191 (+   712us) log.cc:844] Serialized 23 byte log entry
0628 07:52:41.183723 (+   532us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.183793 (+    70us) write_op.cc:454] Released schema lock
0628 07:52:41.183820 (+    27us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:41.209288 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:41.209720 (+   432us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:41.209770 (+    50us) write_op.cc:435] Acquired schema lock
0628 07:52:41.209786 (+    16us) tablet.cc:660] Decoding operations
0628 07:52:41.210065 (+   279us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:41.210112 (+    47us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:41.210140 (+    28us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:41.210299 (+   159us) tablet.cc:699] Row locks acquired
0628 07:52:41.210316 (+    17us) write_op.cc:260] PREPARE: finished
0628 07:52:41.210434 (+   118us) write_op.cc:270] Start()
0628 07:52:41.210550 (+   116us) write_op.cc:276] Timestamp: P: 1751097161210355 usec, L: 0
0628 07:52:41.210590 (+    40us) op_driver.cc:348] REPLICATION: starting
0628 07:52:41.232605 (+ 22015us) log.cc:844] Serialized 356 byte log entry
0628 07:52:41.234058 (+  1453us) spinlock_profiling.cc:243] Waited 21.5 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b135d1 00007f5be8b8ab68 00007f5be8ff9811 00007f5be8ff8fe8 00007f5be8ffdc81 00007f5be8ffdc39 00007f5be8ffdbc9 00007f5be8ffdb91 00007f5be8ffcdbd 000000000059bcd4 000000000059bb09
0628 07:52:41.241846 (+  7788us) op_driver.cc:464] REPLICATION: finished
0628 07:52:41.242200 (+   354us) write_op.cc:301] APPLY: starting
0628 07:52:41.242252 (+    52us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:41.242333 (+    81us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:41.242347 (+    14us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:41.242510 (+   163us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:41.242623 (+   113us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:41.242664 (+    41us) write_op.cc:312] APPLY: finished
0628 07:52:41.242830 (+   166us) log.cc:844] Serialized 23 byte log entry
0628 07:52:41.243361 (+   531us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.243416 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:41.243447 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:41.243748 (+   301us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:41.243780 (+    32us) write_op.cc:454] Released schema lock
Metrics: {"reported_tablets":26,"child_traces":[["op",{"apply.queue_time_us":75,"apply.run_cpu_time_us":1626,"apply.run_wall_time_us":43336,"num_ops":1,"prepare.queue_time_us":3377,"prepare.run_cpu_time_us":4121,"prepare.run_wall_time_us":42200,"replication_time_us":85164,"spinlock_wait_cycles":88965888}],["op",{"apply.queue_time_us":144,"apply.run_cpu_time_us":1026,"apply.run_wall_time_us":1518,"num_ops":1,"prepare.queue_time_us":21892,"prepare.run_cpu_time_us":2973,"prepare.run_wall_time_us":24946,"raft.queue_time_us":157,"raft.run_cpu_time_us":566,"raft.run_wall_time_us":1219,"replication_time_us":31175,"spinlock_wait_cycles":49463552}]]}
I20250628 07:52:42.537911   543 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ca0af8e0e69c4e2ba99ffa3917290323" } modify_external_catalogs: true
I20250628 07:52:42.539072  2237 ts_tablet_manager.cc:1918] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:42.539440  2237 log.cc:1199] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/8f7e8d9640c74c8da5ac00adaf86f8a1
I20250628 07:52:42.539670   758 leader_election.cc:304] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:42.540760  2440 raft_consensus.cc:2802] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:42.541087  2440 raft_consensus.cc:491] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:42.541517  2440 raft_consensus.cc:3058] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.546820  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:42.547356  1125 raft_consensus.cc:2466] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:42.548369   758 leader_election.cc:304] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:42.549155  2441 raft_consensus.cc:2802] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:42.549540  2441 raft_consensus.cc:491] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:42.549860  2441 raft_consensus.cc:3058] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.549015  2440 raft_consensus.cc:513] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.558482  2512 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:52:42.560082  2440 leader_election.cc:290] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.560276  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:42.540128  2237 ts_tablet_manager.cc:1939] T 8f7e8d9640c74c8da5ac00adaf86f8a1 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:42.560789  1125 raft_consensus.cc:3058] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:42.490006   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 75) took 3061 ms (client timeout 1000 ms). Trace:
I20250628 07:52:42.573297  2432 raft_consensus.cc:491] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.575793  2432 raft_consensus.cc:513] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.577008  2428 ts_tablet_manager.cc:1905] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:42.577380  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 371) took 4264 ms. Trace:
I20250628 07:52:42.577708  2237 rpcz_store.cc:276] 0628 07:52:38.312880 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.315931 (+  3051us) service_pool.cc:225] Handling call
0628 07:52:42.096833 (+3780902us) ts_tablet_manager.cc:1145] Deleting tablet 8f7e8d9640c74c8da5ac00adaf86f8a1
0628 07:52:42.096871 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.481987 (+385116us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.538818 (+ 56831us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.577320 (+ 38502us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3660138}
I20250628 07:52:42.579005  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 372) took 4004 ms. Trace:
I20250628 07:52:42.579291  2237 rpcz_store.cc:276] 0628 07:52:38.574290 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:38.574406 (+   116us) service_pool.cc:225] Handling call
0628 07:52:42.578656 (+4004250us) ts_tablet_manager.cc:1145] Deleting tablet e536268f648d4b0da1b9edc338ae974a
0628 07:52:42.578703 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.578952 (+   249us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4003761}
I20250628 07:52:42.580149  2237 tablet_replica.cc:331] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:42.582276   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 8f7e8d9640c74c8da5ac00adaf86f8a1 (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]) successfully deleted
W20250628 07:52:42.582955   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet e536268f648d4b0da1b9edc338ae974a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet e536268f648d4b0da1b9edc338ae974a already in progress: creating tablet
I20250628 07:52:42.603972  1125 raft_consensus.cc:2466] T 1ed24dbf9c234b43b352114d5219bbea P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:42.605101   758 leader_election.cc:304] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:42.607121  1234 tablet_bootstrap.cc:654] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:42.609885  2440 raft_consensus.cc:2802] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:42.607659  2441 raft_consensus.cc:513] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.610522  2440 raft_consensus.cc:695] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:42.611546  2440 consensus_queue.cc:237] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.615916  2441 leader_election.cc:290] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.624946   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 2ae890710f7f44cf899dd19e96fa9181 (table default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]) successfully deleted
W20250628 07:52:42.626045  2598 log.cc:927] Time spent T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4: Append to log took a long time: real 0.192s	user 0.001s	sys 0.000s
I20250628 07:52:42.627890   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:42.573334  2540 consensus_queue.cc:1035] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.062s
I20250628 07:52:42.633129   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "77234b63748849b6996c7aa0ca23ef05" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:42.520645  1230 consensus_queue.cc:260] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.640338  2410 raft_consensus.cc:491] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.640928  2432 leader_election.cc:290] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.640815  2410 raft_consensus.cc:513] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.644500  2410 leader_election.cc:290] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:42.646544   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:42.608428   543 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_ca0af8e0e69c4e2ba99ffa3917290323" } modify_external_catalogs: true
I20250628 07:52:42.648273  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
W20250628 07:52:42.570178   539 rpcz_store.cc:269] 0628 07:52:39.428094 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.444257 (+ 16163us) service_pool.cc:225] Handling call
0628 07:52:42.489895 (+3045638us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:40.861764 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:40.862215 (+   451us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:40.862241 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:40.862256 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:40.862532 (+   276us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:40.862577 (+    45us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:40.862600 (+    23us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:40.862744 (+   144us) tablet.cc:699] Row locks acquired
0628 07:52:40.862759 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:40.862861 (+   102us) write_op.cc:270] Start()
0628 07:52:40.862945 (+    84us) write_op.cc:276] Timestamp: P: 1751097160862817 usec, L: 0
0628 07:52:40.862982 (+    37us) op_driver.cc:348] REPLICATION: starting
0628 07:52:40.863483 (+   501us) log.cc:844] Serialized 356 byte log entry
0628 07:52:40.930275 (+ 66792us) op_driver.cc:464] REPLICATION: finished
0628 07:52:40.930641 (+   366us) write_op.cc:301] APPLY: starting
0628 07:52:40.930686 (+    45us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:40.930768 (+    82us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:40.930843 (+    75us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:40.931020 (+   177us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:40.931133 (+   113us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:40.931170 (+    37us) write_op.cc:312] APPLY: finished
0628 07:52:40.931323 (+   153us) log.cc:844] Serialized 23 byte log entry
0628 07:52:40.931474 (+   151us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.931527 (+    53us) write_op.cc:454] Released schema lock
0628 07:52:40.931576 (+    49us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:40.931800 (+   224us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:40.931831 (+    31us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":106716,"reported_tablets":45,"child_traces":[["op",{"apply.queue_time_us":239,"apply.run_cpu_time_us":1130,"apply.run_wall_time_us":1129,"num_ops":1,"prepare.queue_time_us":1061,"prepare.run_cpu_time_us":1935,"prepare.run_wall_time_us":1935,"replication_time_us":67192}]]}
W20250628 07:52:42.655293   920 leader_election.cc:343] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.648911  1121 raft_consensus.cc:2466] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:42.657430   921 leader_election.cc:304] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:42.660828  2536 raft_consensus.cc:2802] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:42.664968  2536 raft_consensus.cc:491] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:42.665351  2536 raft_consensus.cc:3058] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.669123  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:42.670015  1125 raft_consensus.cc:3058] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.674777  2536 raft_consensus.cc:513] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.681841   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:42.683449   920 leader_election.cc:343] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.687597  2540 consensus_queue.cc:1035] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:52:42.696247   921 consensus_peers.cc:489] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:42.712731  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 89750.4us
I20250628 07:52:42.714490   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:42.715422  2591 tablet.cc:1722] T bed1ea07e3d44c08adf37ced9dbef85e P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20250628 07:52:42.719345  2237 raft_consensus.cc:2241] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:42.719863  2237 raft_consensus.cc:2270] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:42.721356  1229 tablet_bootstrap.cc:492] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:42.722004  2237 ts_tablet_manager.cc:1905] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:42.722046  1229 ts_tablet_manager.cc:1397] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.300s	user 0.009s	sys 0.014s
I20250628 07:52:42.728324  1230 ts_tablet_manager.cc:1428] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.292s	user 0.002s	sys 0.005s
W20250628 07:52:42.738536   758 consensus_peers.cc:489] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c4fcefaa9e954b52bbbee6e0a2f2f939. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:42.739269  1062 leader_election.cc:343] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.743943  1234 tablet_bootstrap.cc:492] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:42.744314  1234 ts_tablet_manager.cc:1397] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.244s	user 0.010s	sys 0.000s
I20250628 07:52:42.748703  1234 raft_consensus.cc:357] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.749272  1234 raft_consensus.cc:383] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.750036  1234 raft_consensus.cc:738] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.752183  2428 ts_tablet_manager.cc:1918] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:42.752564  2428 log.cc:1199] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fa3ec14241e344029df3e39473181078
I20250628 07:52:42.753264  2428 ts_tablet_manager.cc:1939] T fa3ec14241e344029df3e39473181078 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:42.753966  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 420) took 1337 ms. Trace:
I20250628 07:52:42.754244  2428 rpcz_store.cc:276] 0628 07:52:41.416574 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.416723 (+   149us) service_pool.cc:225] Handling call
0628 07:52:42.421540 (+1004817us) ts_tablet_manager.cc:1145] Deleting tablet fa3ec14241e344029df3e39473181078
0628 07:52:42.421590 (+    50us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.617930 (+196340us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.751971 (+134041us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.753919 (+  1948us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":975595}
I20250628 07:52:42.755316  2428 tablet_replica.cc:331] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:42.759200  2237 ts_tablet_manager.cc:1918] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:42.759603  2237 log.cc:1199] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/de7ff3260b0049d682dba82f914474de
I20250628 07:52:42.761168   540 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 88d4a84c8ed7475c9bf043aaba8ad624
I20250628 07:52:42.763298  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 with delete_type TABLET_DATA_DELETED (replaced by tablet 71d772c683f94e79bd14bfc139357844) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.763933   540 catalog_manager.cc:6652] ReplaceTablet: tablet 88d4a84c8ed7475c9bf043aaba8ad624 deleted and replaced by tablet 71d772c683f94e79bd14bfc139357844
I20250628 07:52:42.772766  2428 raft_consensus.cc:2241] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:42.773303  2428 raft_consensus.cc:2270] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:42.776017  1230 tablet_bootstrap.cc:492] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:42.765370   789 tablet_service.cc:1515] Processing DeleteTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 with delete_type TABLET_DATA_DELETED (replaced by tablet 71d772c683f94e79bd14bfc139357844) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:42.760278  2237 ts_tablet_manager.cc:1939] T de7ff3260b0049d682dba82f914474de P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:42.781064  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 374) took 3529 ms. Trace:
I20250628 07:52:42.781378  2237 rpcz_store.cc:276] 0628 07:52:39.251675 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.251762 (+    87us) service_pool.cc:225] Handling call
0628 07:52:42.580021 (+3328259us) ts_tablet_manager.cc:1145] Deleting tablet de7ff3260b0049d682dba82f914474de
0628 07:52:42.580055 (+    34us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.728896 (+148841us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.758989 (+ 30093us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.781011 (+ 22022us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":138382,"tablet-delete.queue_time_us":3327725}
I20250628 07:52:42.783435   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet de7ff3260b0049d682dba82f914474de (table default.Table_e13a07ffb7754b44a1e224a0cc1e4289 [id=3594565eae314c6e861c681fb61092bf]) successfully deleted
W20250628 07:52:42.782761  2585 tablet_replica.cc:1408] Time spent flushing the Log queue took a long time: real 0.391s	user 0.006s	sys 0.011s
I20250628 07:52:42.795361  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 375) took 3401 ms. Trace:
I20250628 07:52:42.795775  2237 rpcz_store.cc:276] 0628 07:52:39.393831 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.393931 (+   100us) service_pool.cc:225] Handling call
0628 07:52:42.795047 (+3401116us) ts_tablet_manager.cc:1145] Deleting tablet 35fe2e8156cc43d7b3c02092877063b6
0628 07:52:42.795088 (+    41us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.795309 (+   221us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3400597}
W20250628 07:52:42.796758   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 35fe2e8156cc43d7b3c02092877063b6 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 35fe2e8156cc43d7b3c02092877063b6 already in progress: creating tablet
I20250628 07:52:42.797176  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 376) took 3339 ms. Trace:
I20250628 07:52:42.797451  2237 rpcz_store.cc:276] 0628 07:52:39.457829 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:39.458622 (+   793us) service_pool.cc:225] Handling call
0628 07:52:42.796842 (+3338220us) ts_tablet_manager.cc:1145] Deleting tablet d2578fbf2e974cc0bca65e32f45694ab
0628 07:52:42.796881 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.797131 (+   250us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3337720}
I20250628 07:52:42.798804   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_743f866ef4ae435180af5ecffcb04bc2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:42.800370   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_743f866ef4ae435180af5ecffcb04bc2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.807734  2428 ts_tablet_manager.cc:1905] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:42.800757  2237 tablet_replica.cc:331] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:42.816315  2237 raft_consensus.cc:2241] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:42.817080  2237 raft_consensus.cc:2270] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:42.821179   539 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_4bed6916b6b14023b7099f15be8c1edc"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:42.822860   539 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4bed6916b6b14023b7099f15be8c1edc in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.825349   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_c35cbc8840ec40728f62ffc4fb20ce69"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:42.827055   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c35cbc8840ec40728f62ffc4fb20ce69 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.811762  2536 leader_election.cc:290] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.831944  1125 raft_consensus.cc:2466] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:42.833050   758 leader_election.cc:304] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:42.833858  2441 raft_consensus.cc:2802] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:42.834252  2441 raft_consensus.cc:695] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:42.835470  2441 consensus_queue.cc:237] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.838635  2237 ts_tablet_manager.cc:1905] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:42.841811  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:42.842314  1128 raft_consensus.cc:3058] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:42.850677  1099 tablet_service.cc:1515] Processing DeleteTablet for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 with delete_type TABLET_DATA_DELETED (replaced by tablet 4eac11942d104acab461d483252128c5) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:42.854820  1229 raft_consensus.cc:357] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.859791  1229 raft_consensus.cc:383] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:42.860067  1229 raft_consensus.cc:738] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:42.860603  1229 consensus_queue.cc:260] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.856669  2618 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:42.861663   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:42.861666  2618 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:42.864190   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:42.852715   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:42.866262   634 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:42.866766   982 raft_consensus.cc:2466] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:42.867769  2059 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:42.751258  1234 consensus_queue.cc:260] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.874936  2614 tablet.cc:1722] T bfc303c4588d44428984d8203fc05185 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:42.879732  1229 ts_tablet_manager.cc:1428] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.157s	user 0.005s	sys 0.000s
I20250628 07:52:42.884063   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a447edf0960c4739930450a8c24ff6f7" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:42.892241   958 tablet_service.cc:1515] Processing DeleteTablet for tablet 22b47b39f8eb45a0b5d2c5348bfbc55c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:40 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.898036   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1ed24dbf9c234b43b352114d5219bbea" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:42.892974  1229 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:42.898912   981 raft_consensus.cc:3058] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:42.902653   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1efec32bd8cd468eb13825d51722f7cc consensus state has no leader
I20250628 07:52:42.904534  2609 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.905066  2609 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.908458   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:42.909654  1062 leader_election.cc:343] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.910521  1128 raft_consensus.cc:2466] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:42.911178  2609 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:42.911592   921 leader_election.cc:304] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:42.912863   966 tablet_service.cc:1515] Processing DeleteTablet for tablet ee03996105074308bda56d36ba3a1c5d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:24 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.913048   956 tablet_service.cc:1515] Processing DeleteTablet for tablet fd67d7d9ba9446c19f1711fb5bd73ebd with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:41 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.913902   957 tablet_service.cc:1515] Processing DeleteTablet for tablet c3d1bc8483374a2a917451a43b3d42ed with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.914108   963 tablet_service.cc:1515] Processing DeleteTablet for tablet d87dd96d00964e12acd6e380a62c94d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:38 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.914698   958 tablet_service.cc:1515] Processing DeleteTablet for tablet 8155990eebd84fcf9ebc357e186d1bf2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:28 UTC) from {username='slave'} at 127.0.0.1:35848
W20250628 07:52:42.914845  1062 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.915169  2568 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:35848 (request call id 413) took 1696 ms. Trace:
I20250628 07:52:42.915221  1062 leader_election.cc:304] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:42.915355   547 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:42.916175  2606 raft_consensus.cc:491] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:42.916818  2609 raft_consensus.cc:2747] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:42.916585  2606 raft_consensus.cc:513] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:42.917392   547 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:42.918358  2606 leader_election.cc:290] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:42.919803  1062 leader_election.cc:343] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.915483  2568 rpcz_store.cc:276] 0628 07:52:41.198608 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.198703 (+    95us) service_pool.cc:225] Handling call
0628 07:52:42.894454 (+1695751us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:41.461978 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
0628 07:52:41.462261 (+   283us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
0628 07:52:41.462307 (+    46us) alter_schema_op.cc:66] Acquired schema lock
0628 07:52:41.462357 (+    50us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
0628 07:52:41.462458 (+   101us) alter_schema_op.cc:128] START. Timestamp: 1751097161462373
0628 07:52:41.462477 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:41.475237 (+ 12760us) log.cc:844] Serialized 173 byte log entry
0628 07:52:42.102746 (+627509us) op_driver.cc:464] REPLICATION: finished
0628 07:52:42.430612 (+327866us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
0628 07:52:42.431988 (+  1376us) log.cc:844] Serialized 5 byte log entry
0628 07:52:42.775601 (+343613us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.880514 (+104913us) log.cc:844] Serialized 14 byte log entry
0628 07:52:42.889456 (+  8942us) alter_schema_op.cc:73] Released schema lock
0628 07:52:42.890453 (+   997us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
Metrics: {"child_traces":[["op",{"apply.queue_time_us":327616,"mutex_wait_us":230849,"prepare.queue_time_us":200142,"prepare.run_cpu_time_us":14894,"prepare.run_wall_time_us":124680,"raft.queue_time_us":58500,"raft.run_cpu_time_us":1455,"raft.run_wall_time_us":10598,"replication_time_us":640123,"thread_start_us":394015,"threads_started":3,"wal-append.queue_time_us":239245}]]}
I20250628 07:52:42.920181  1062 leader_election.cc:304] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:42.920177   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:42.920997  2606 raft_consensus.cc:2747] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:42.921051  2568 tablet.cc:1722] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20250628 07:52:42.924054  1062 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.927903   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fa3ec14241e344029df3e39473181078 (table default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]) successfully deleted
W20250628 07:52:42.930284   757 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:42.931152   533 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 6fdda1e4f7af468982a813f1ac023e01 (table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 6fdda1e4f7af468982a813f1ac023e01 consensus state has no leader
W20250628 07:52:42.935143   757 leader_election.cc:343] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:42.936246   956 tablet_service.cc:1515] Processing DeleteTablet for tablet 88d4a84c8ed7475c9bf043aaba8ad624 with delete_type TABLET_DATA_DELETED (replaced by tablet 71d772c683f94e79bd14bfc139357844) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:42.938095  2526 raft_consensus.cc:2802] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:42.938513  2526 raft_consensus.cc:695] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:42.939271  2620 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:42.939229  2526 consensus_queue.cc:237] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:42.937206  2618 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
W20250628 07:52:42.949918   544 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 71c5bdface3d44ecae33bef018e7f18d consensus state has no leader
W20250628 07:52:42.955442  2059 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.188:42677, user_credentials={real_user=slave}} blocked reactor thread for 87695us
I20250628 07:52:42.933019  2591 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:35848 (request call id 415) took 1669 ms. Trace:
W20250628 07:52:42.969130   878 tablet.cc:2378] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:52:42.859644  2514 ts_tablet_manager.cc:1918] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:42.982551  2514 log.cc:1199] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/af13b56df157443c994f747e52b0f458
I20250628 07:52:42.983628  2514 ts_tablet_manager.cc:1939] T af13b56df157443c994f747e52b0f458 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:42.873157  1234 ts_tablet_manager.cc:1428] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.129s	user 0.008s	sys 0.000s
I20250628 07:52:43.009737  1234 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:43.029601  1230 tablet_bootstrap.cc:654] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:43.033958   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 76) took 2071 ms (client timeout 1000 ms). Trace:
W20250628 07:52:43.034289   546 rpcz_store.cc:269] 0628 07:52:40.962190 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.995996 (+ 33806us) service_pool.cc:225] Handling call
0628 07:52:41.131071 (+135075us) spinlock_profiling.cc:243] Waited 104 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:43.033885 (+1902814us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":170758,"reported_tablets":49,"spinlock_wait_cycles":238840064}
I20250628 07:52:43.039084  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 394) took 1717 ms. Trace:
I20250628 07:52:43.039518  2514 rpcz_store.cc:276] 0628 07:52:41.321789 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.322233 (+   444us) service_pool.cc:225] Handling call
0628 07:52:42.149294 (+827061us) ts_tablet_manager.cc:1145] Deleting tablet af13b56df157443c994f747e52b0f458
0628 07:52:42.325911 (+176617us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.599001 (+273090us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.859390 (+260389us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.039012 (+179622us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":1846,"tablet-delete.queue_time_us":826472}
I20250628 07:52:43.042147   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:43.043339  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 396) took 1523 ms. Trace:
I20250628 07:52:42.961283  2591 rpcz_store.cc:276] 0628 07:52:41.263212 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.263316 (+   104us) service_pool.cc:225] Handling call
0628 07:52:41.556176 (+292860us) spinlock_profiling.cc:243] Waited 4.04 ms on lock 0x7b5800000098. stack: 00007ff6ffd094b3 00007ff6ffd093b0 00007ff6ff885574 00000000004fa174 00000000004fa109 00007ff6ffb7e9b1 00007ff6ffcfdee1 00007ff6ffcfdd92 00007ff7089748f9 00007ff707bfcac0 00007ff707cc37bb 00007ff707cc3841 00007ff707c81f80 00007ff707c6be69 00007ff707c6b4b4 00007ff70633f834
0628 07:52:42.932944 (+1376768us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:41.563436 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
0628 07:52:41.565338 (+  1902us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
0628 07:52:41.565372 (+    34us) alter_schema_op.cc:66] Acquired schema lock
0628 07:52:41.566183 (+   811us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
0628 07:52:41.566281 (+    98us) alter_schema_op.cc:128] START. Timestamp: 1751097161566188
0628 07:52:41.566296 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:41.566779 (+   483us) log.cc:844] Serialized 173 byte log entry
0628 07:52:42.348736 (+781957us) op_driver.cc:464] REPLICATION: finished
0628 07:52:42.715199 (+366463us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
0628 07:52:42.716916 (+  1717us) log.cc:844] Serialized 5 byte log entry
0628 07:52:42.931433 (+214517us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:42.932229 (+   796us) log.cc:844] Serialized 14 byte log entry
0628 07:52:42.932813 (+   584us) alter_schema_op.cc:73] Released schema lock
0628 07:52:42.932842 (+    29us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
0628 07:52:42.953936 (+ 21094us) spinlock_profiling.cc:243] Waited 83.3 ms on lock 0x7b54000848d4. stack: 00007ff6ffd094b3 00007ff6ffd093b0 00007ff6ff885574 00000000004fa174 00000000004fa109 00007ff706c0b5d1 00007ff706c8406a 00007ff706c21534 00007ff706c214e3 00007ff706c21458 00007ff706c21413 00007ff706c206fc 00007ff706c4a226 00007ff706c321eb 00007ff706c1c9bd 00007ff706c20395
Metrics: {"spinlock_wait_cycles":9285888,"child_traces":[["op",{"apply.queue_time_us":366298,"mutex_wait_us":96867,"prepare.queue_time_us":131,"prepare.run_cpu_time_us":110328,"prepare.run_wall_time_us":806447,"raft.queue_time_us":129579,"raft.run_cpu_time_us":30287,"raft.run_wall_time_us":748927,"replication_time_us":782323,"spinlock_wait_cycles":191502720,"thread_start_us":1180397,"threads_started":2,"wal-append.queue_time_us":590836}]]}
I20250628 07:52:42.962363  2533 raft_consensus.cc:491] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.053498  2533 raft_consensus.cc:513] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.056056  2533 leader_election.cc:290] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:42.965073   981 raft_consensus.cc:2466] T a447edf0960c4739930450a8c24ff6f7 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:43.008981   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.018496   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.017865   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:43.059994   920 leader_election.cc:343] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Not found: Tablet not found: 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:42.982657  1229 tablet_bootstrap.cc:654] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:43.061326   921 leader_election.cc:343] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:43.061692   921 leader_election.cc:304] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:43.062595  2533 raft_consensus.cc:2747] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:42.949075  2585 tablet.cc:1722] T 46e0aa090f544e8aa0a426c61925f345 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:43.071041  2591 tablet.cc:1722] T fceb8ef4ddb44e7eb11be71a5087bd27 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:43.083814  1234 tablet_bootstrap.cc:654] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.105646  2428 ts_tablet_manager.cc:1918] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:43.106068  2428 log.cc:1199] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/63115b7be4a94b639c18601d438ed947
I20250628 07:52:43.106734  2428 ts_tablet_manager.cc:1939] T 63115b7be4a94b639c18601d438ed947 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:43.120843  1229 tablet_bootstrap.cc:492] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
W20250628 07:52:43.073201  1062 leader_election.cc:343] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.089893  1230 tablet_bootstrap.cc:492] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:43.129765  1230 ts_tablet_manager.cc:1397] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.400s	user 0.010s	sys 0.000s
I20250628 07:52:43.129601  1062 leader_election.cc:304] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.129643  2514 rpcz_store.cc:276] 0628 07:52:41.519962 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.520286 (+   324us) service_pool.cc:225] Handling call
0628 07:52:43.042933 (+1522647us) ts_tablet_manager.cc:1145] Deleting tablet 06b9b63649384313948964b3c0dd11cf
0628 07:52:43.042986 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.043282 (+   296us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1522099}
W20250628 07:52:43.130440  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 57273.5us
W20250628 07:52:43.131201   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 06b9b63649384313948964b3c0dd11cf because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 06b9b63649384313948964b3c0dd11cf already in progress: creating tablet
W20250628 07:52:43.131392  1062 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.131724  1062 leader_election.cc:304] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.132274  2514 tablet_replica.cc:331] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:43.132550  1062 leader_election.cc:343] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.132593  1230 raft_consensus.cc:357] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.133275  1230 raft_consensus.cc:383] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.133468  2612 raft_consensus.cc:2747] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.133535  1230 raft_consensus.cc:738] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.129079  2606 raft_consensus.cc:491] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.133765  2514 raft_consensus.cc:2241] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.134143  2612 raft_consensus.cc:2747] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.134245   544 master_service.cc:948] ReplaceTablet: received request to replace tablet 51a44ec161684f74b228830ff06f1957 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:43.134078  2606 raft_consensus.cc:513] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.134661  2514 raft_consensus.cc:2270] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.132929  1062 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.135892  2606 leader_election.cc:290] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.135937  2609 raft_consensus.cc:2747] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.136824  1230 consensus_queue.cc:260] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.137351   538 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_50f3c0fe4d0445c7b1cc05590d41a738" } new_table_name: "default.Table_948ee6f5274e4638b253dcf9f5df2e31" modify_external_catalogs: true
I20250628 07:52:43.138058  2609 raft_consensus.cc:491] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.138410  2609 raft_consensus.cc:513] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.140090  2609 leader_election.cc:290] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.140898   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:43.144001  1062 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.143961   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "296d777dcbfb4d5aa80db78e3f07426c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.143474  2103 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" instance_seqno: 1751097128303340) as {username='slave'} at 127.0.107.130:40999; Asking this server to re-register.
W20250628 07:52:43.144837  1062 leader_election.cc:343] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.146309  1230 ts_tablet_manager.cc:1428] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.016s	user 0.005s	sys 0.000s
I20250628 07:52:43.147174  1230 tablet_bootstrap.cc:492] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:43.152241  1230 tablet_bootstrap.cc:654] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.161446   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.129419  1229 ts_tablet_manager.cc:1397] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.237s	user 0.011s	sys 0.000s
I20250628 07:52:43.163786   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "296d777dcbfb4d5aa80db78e3f07426c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.164175  1229 raft_consensus.cc:357] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.164824  1229 raft_consensus.cc:383] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.165063  1229 raft_consensus.cc:738] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.165611  1229 consensus_queue.cc:260] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.167574  1229 ts_tablet_manager.cc:1428] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.005s	user 0.003s	sys 0.000s
W20250628 07:52:43.167526  1062 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.167959  1062 leader_election.cc:304] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.168632  2606 raft_consensus.cc:2747] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:43.170262  1062 leader_election.cc:343] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:43.170419   757 consensus_peers.cc:489] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 63115b7be4a94b639c18601d438ed947. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:43.170619  1062 leader_election.cc:304] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.171195  2606 raft_consensus.cc:2747] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.109076  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 417) took 1837 ms. Trace:
I20250628 07:52:43.177579  1234 tablet_bootstrap.cc:492] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:43.178004  1234 ts_tablet_manager.cc:1397] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.168s	user 0.009s	sys 0.000s
I20250628 07:52:43.177544  2428 rpcz_store.cc:276] 0628 07:52:41.271661 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.341613 (+ 69952us) service_pool.cc:225] Handling call
0628 07:52:42.755187 (+1413574us) ts_tablet_manager.cc:1145] Deleting tablet 63115b7be4a94b639c18601d438ed947
0628 07:52:42.755223 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:42.902644 (+147421us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.105409 (+202765us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.109021 (+  3612us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":16664,"tablet-delete.queue_time_us":1281611}
I20250628 07:52:43.178817   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 63115b7be4a94b639c18601d438ed947 (table default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]) successfully deleted
I20250628 07:52:43.180428  1234 raft_consensus.cc:357] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.181071  1229 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:43.181069  1234 raft_consensus.cc:383] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.181368  1234 raft_consensus.cc:738] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.182044  1234 consensus_queue.cc:260] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.183859  1033 heartbeater.cc:461] Registering TS with master...
I20250628 07:52:43.186322  1230 tablet_bootstrap.cc:492] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
W20250628 07:52:43.186126   436 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerService.ListTablets -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 76018.7us
I20250628 07:52:43.186728  1230 ts_tablet_manager.cc:1397] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.040s	user 0.004s	sys 0.004s
I20250628 07:52:43.168865  2514 ts_tablet_manager.cc:1905] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.203660  1229 tablet_bootstrap.cc:654] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.209697  1230 raft_consensus.cc:357] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.210402  1230 raft_consensus.cc:383] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.210685  1230 raft_consensus.cc:738] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.213446  2526 consensus_queue.cc:1035] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:43.213842  2428 tablet_replica.cc:331] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:43.214537  2428 raft_consensus.cc:2241] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.214897  2428 raft_consensus.cc:2270] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.216496  2428 ts_tablet_manager.cc:1905] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.218463  2594 raft_consensus.cc:491] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.218919  2594 raft_consensus.cc:513] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.220629  2594 leader_election.cc:290] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:43.220458  1230 consensus_queue.cc:260] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.222738  1234 ts_tablet_manager.cc:1428] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.044s	user 0.006s	sys 0.000s
I20250628 07:52:43.223562  1234 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:43.231863  1230 ts_tablet_manager.cc:1428] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.045s	user 0.006s	sys 0.001s
I20250628 07:52:43.232765  1230 tablet_bootstrap.cc:492] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:43.233563   785 rpcz_store.cc:275] Call kudu.tserver.TabletServerService.ListTablets from 127.0.0.1:41476 (request call id 55) took 1701 ms. Trace:
I20250628 07:52:43.233963   785 rpcz_store.cc:276] 0628 07:52:41.532223 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.532318 (+    95us) service_pool.cc:225] Handling call
0628 07:52:41.712923 (+180605us) spinlock_profiling.cc:243] Waited 92.8 ms on lock 0x7b5c0002290c. stack: 00007f81877f84b3 00007f81877f83b0 00007f8187374574 00000000004fa174 00000000004fa109 00007f818eb14d11 00007f818ecb51e9 00007f818eba067c 00007f818f76ab70 00007f818dfe0244 00007f818dfe01d2 00007f818dfe0101 00007f818dfe007c 00007f818dfdf332 00007f818c53edcc 00007f818c53e206
0628 07:52:42.509708 (+796785us) spinlock_profiling.cc:243] Waited 116 ms on lock 0x7b54000e2ed4. stack: 00007f81877f84b3 00007f81877f83b0 00007f8187374574 00000000004fa174 00000000004fa109 00007f818e6fa5d1 00007f818e76fc38 00007f818f76ab9d 00007f818dfe0244 00007f818dfe01d2 00007f818dfe0101 00007f818dfe007c 00007f818dfdf332 00007f818c53edcc 00007f818c53e206 00007f818c53dbaf
0628 07:52:42.737930 (+228222us) spinlock_profiling.cc:243] Waited 110 ms on lock 0x7b54000e33d4. stack: 00007f81877f84b3 00007f81877f83b0 00007f8187374574 00000000004fa174 00000000004fa109 00007f818e6fa5d1 00007f818e76fc38 00007f818f76ab9d 00007f818dfe0244 00007f818dfe01d2 00007f818dfe0101 00007f818dfe007c 00007f818dfdf332 00007f818c53edcc 00007f818c53e206 00007f818c53dbaf
0628 07:52:42.839825 (+101895us) spinlock_profiling.cc:243] Waited 63.6 ms on lock 0x7b5400104354. stack: 00007f81877f84b3 00007f81877f83b0 00007f8187374574 00000000004fa174 00000000004fa109 00007f818e6fa5d1 00007f818e76fc38 00007f818f76ab9d 00007f818dfe0244 00007f818dfe01d2 00007f818dfe0101 00007f818dfe007c 00007f818dfdf332 00007f818c53edcc 00007f818c53e206 00007f818c53dbaf
0628 07:52:43.233376 (+393551us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":879890560}
I20250628 07:52:43.242331  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:43.243106  1126 raft_consensus.cc:2391] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:43.245388   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1fe54aa8c2a04c238afb82ef9ff7fa81" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.247153  1229 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.247227   757 leader_election.cc:304] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.247555  1229 ts_tablet_manager.cc:1397] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.067s	user 0.006s	sys 0.003s
I20250628 07:52:43.248745  2441 raft_consensus.cc:3058] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.249783  1229 raft_consensus.cc:357] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.250391  1229 raft_consensus.cc:383] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.250650  1229 raft_consensus.cc:738] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.251183  1229 consensus_queue.cc:260] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.253943  1230 tablet_bootstrap.cc:654] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.256397  2441 raft_consensus.cc:2747] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.253059  1229 ts_tablet_manager.cc:1428] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.005s	user 0.000s	sys 0.003s
I20250628 07:52:43.262666  1229 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:43.267836  1229 tablet_bootstrap.cc:654] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.269134  2337 consensus_queue.cc:1035] T 8226c926201040c993804d2ae8450202 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:43.257405  2428 ts_tablet_manager.cc:1918] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:43.273649  2428 log.cc:1199] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:43.274466  2428 ts_tablet_manager.cc:1939] T 8e3e0e52b4924e67904d3acfa4dd5752 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:43.276664  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 421) took 1819 ms. Trace:
I20250628 07:52:43.276991  2428 rpcz_store.cc:276] 0628 07:52:41.456808 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.457144 (+   336us) service_pool.cc:225] Handling call
0628 07:52:43.178514 (+1721370us) ts_tablet_manager.cc:1145] Deleting tablet 8e3e0e52b4924e67904d3acfa4dd5752
0628 07:52:43.213718 (+ 35204us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.225270 (+ 11552us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.257187 (+ 31917us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.276611 (+ 19424us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1686337}
W20250628 07:52:43.283561   921 consensus_peers.cc:489] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:43.282943   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 8e3e0e52b4924e67904d3acfa4dd5752 (table default.Table_7e69244decba48b8b9a9f406ccf40827 [id=3d33e99eb2eb44b0944fc99e9aa8191b]) successfully deleted
I20250628 07:52:43.294772   540 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_f6dc476a6f3b42afa13683cc9611a247"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:52:43.296247  2634 tablet.cc:1722] T 2f8099bd0c9649b492edd8b657fd19e3 P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:43.301869  2585 tablet.cc:1722] T 2f8099bd0c9649b492edd8b657fd19e3 P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:43.312170   430 master-stress-test.cc:577] Killing test thread 1/10
I20250628 07:52:43.317845  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 423) took 1802 ms. Trace:
I20250628 07:52:43.318207  2428 rpcz_store.cc:276] 0628 07:52:41.515183 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.515276 (+    93us) service_pool.cc:225] Handling call
0628 07:52:43.302695 (+1787419us) ts_tablet_manager.cc:1145] Deleting tablet 46ff641001cd4b01a4c09c0d8b79ebf3
0628 07:52:43.317589 (+ 14894us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.317777 (+   188us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":2432,"tablet-delete.queue_time_us":1763980}
I20250628 07:52:43.319983   547 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:47964:
table { table_name: "default.Table_4ed2c883318848a19e2dac4d9820b2e3" } new_table_name: "default.Table_998846661d2441bfba53057917d7f1bf" modify_external_catalogs: true
W20250628 07:52:43.325383   538 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 58d853d69a5145cdabb33437463d4aa2 consensus state has no leader
I20250628 07:52:43.326220   538 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47964:
name: "default.Table_3f9fa575c08344b9965c9fb9bccc8c85"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:52:43.327572   538 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3f9fa575c08344b9965c9fb9bccc8c85 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:52:43.319379   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because the tablet was not found. No further retry: Not found: Tablet not found: 46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:43.339360  2237 ts_tablet_manager.cc:1918] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:43.339707  2237 log.cc:1199] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/d87dd96d00964e12acd6e380a62c94d8
I20250628 07:52:43.340390  2441 raft_consensus.cc:491] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.340955  2441 raft_consensus.cc:513] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.343063  2441 leader_election.cc:290] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:43.343753   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.348734  2606 raft_consensus.cc:491] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.349182  2606 raft_consensus.cc:513] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.350836  2606 leader_election.cc:290] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.354477  2636 tablet.cc:1722] T 2f8099bd0c9649b492edd8b657fd19e3 P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:43.340407  2237 ts_tablet_manager.cc:1939] T d87dd96d00964e12acd6e380a62c94d8 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:43.363037  2428 tablet_replica.cc:331] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:43.363890  2428 raft_consensus.cc:2241] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.364378  2428 raft_consensus.cc:2270] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.365603  2526 raft_consensus.cc:491] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.366617  2526 raft_consensus.cc:513] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.367321  2428 ts_tablet_manager.cc:1905] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.368993   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.369131  2526 leader_election.cc:290] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:43.369928   920 leader_election.cc:343] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.374683  2337 consensus_queue.cc:1035] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:43.377394  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:43.380168  1128 raft_consensus.cc:2466] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:43.384960   921 leader_election.cc:304] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:43.385458  2533 consensus_queue.cc:1035] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:43.385898  2337 raft_consensus.cc:2802] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:43.386881  2337 raft_consensus.cc:491] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:43.387123  1229 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.387193  2337 raft_consensus.cc:3058] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.387554  1229 ts_tablet_manager.cc:1397] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.125s	user 0.018s	sys 0.009s
I20250628 07:52:43.391350   978 raft_consensus.cc:3058] T 1ed24dbf9c234b43b352114d5219bbea P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.392136  1230 tablet_bootstrap.cc:492] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:43.392570  1230 ts_tablet_manager.cc:1397] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.160s	user 0.018s	sys 0.015s
I20250628 07:52:43.394999  1230 raft_consensus.cc:357] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.395641  1230 raft_consensus.cc:383] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.395879  1230 raft_consensus.cc:738] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:43.399447   540 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f6dc476a6f3b42afa13683cc9611a247 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:52:43.400308  2441 consensus_queue.cc:1035] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.003s
W20250628 07:52:43.404273   547 catalog_manager.cc:5842] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 81d9066872654fceb44b7fb63b60f070 consensus state has no leader
I20250628 07:52:43.407657   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.378340  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 404) took 2725 ms. Trace:
I20250628 07:52:43.408226  2237 rpcz_store.cc:276] 0628 07:52:40.653392 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.653507 (+   115us) service_pool.cc:225] Handling call
0628 07:52:42.800612 (+2147105us) ts_tablet_manager.cc:1145] Deleting tablet d87dd96d00964e12acd6e380a62c94d8
0628 07:52:42.800647 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.121628 (+320981us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.339134 (+217506us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.378287 (+ 39153us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2146553}
I20250628 07:52:43.409703   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet d87dd96d00964e12acd6e380a62c94d8 (table default.Table_10140fc26c314afab000f5f4626f9110 [id=a58302bfde464e91b75b443b8a701d24]) successfully deleted
I20250628 07:52:43.410832   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.396579  1230 consensus_queue.cc:260] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.413995  2337 raft_consensus.cc:513] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.415129  1229 raft_consensus.cc:357] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.415808  1229 raft_consensus.cc:383] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.416113  1229 raft_consensus.cc:738] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.416342   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:43.417008  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:43.417544  1125 raft_consensus.cc:3058] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.421432  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 406) took 2721 ms. Trace:
I20250628 07:52:43.414510  1230 ts_tablet_manager.cc:1428] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.022s	user 0.006s	sys 0.000s
W20250628 07:52:43.417440   920 leader_election.cc:343] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.426862  1229 consensus_queue.cc:260] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.431466  2103 ts_manager.cc:194] Registered new tserver with Master: 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:43.434425  2606 raft_consensus.cc:491] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.434903  2606 raft_consensus.cc:513] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.437263  2606 leader_election.cc:290] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.440490  2237 rpcz_store.cc:276] 0628 07:52:40.699997 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.700108 (+   111us) service_pool.cc:225] Handling call
0628 07:52:43.421161 (+2721053us) ts_tablet_manager.cc:1145] Deleting tablet fef70db0996f48c69574337f8f7ecf37
0628 07:52:43.421219 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.421381 (+   162us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2720522}
W20250628 07:52:43.441735   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fef70db0996f48c69574337f8f7ecf37 because the tablet was not found. No further retry: Not found: Tablet not found: fef70db0996f48c69574337f8f7ecf37
I20250628 07:52:43.442685  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 408) took 2738 ms. Trace:
I20250628 07:52:43.443018  2237 rpcz_store.cc:276] 0628 07:52:40.704392 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.709191 (+  4799us) service_pool.cc:225] Handling call
0628 07:52:43.442449 (+2733258us) ts_tablet_manager.cc:1145] Deleting tablet 8f7dde9b060f4d839724e5f718519fb1
0628 07:52:43.442487 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.442637 (+   150us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2732669}
I20250628 07:52:43.424070  2428 ts_tablet_manager.cc:1918] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
W20250628 07:52:43.444118   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 8f7dde9b060f4d839724e5f718519fb1 because the tablet was not found. No further retry: Not found: Tablet not found: 8f7dde9b060f4d839724e5f718519fb1
I20250628 07:52:43.444377  2428 log.cc:1199] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c4fcefaa9e954b52bbbee6e0a2f2f939
I20250628 07:52:43.444990  2237 tablet_replica.cc:331] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:43.427527  1234 tablet_bootstrap.cc:654] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.445168  2428 ts_tablet_manager.cc:1939] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:43.448154  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "29a77cd9dc244d239a518e004c49e3c8" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:43.448863  1121 raft_consensus.cc:2391] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:43.469216  1229 ts_tablet_manager.cc:1428] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.081s	user 0.009s	sys 0.001s
I20250628 07:52:43.477421  2606 raft_consensus.cc:491] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.477551   758 leader_election.cc:304] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.477754   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.477859  2606 raft_consensus.cc:513] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.478977  2440 raft_consensus.cc:3058] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.479426   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.479550  2606 leader_election.cc:290] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:43.483064  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 426) took 1949 ms. Trace:
I20250628 07:52:43.483399  2428 rpcz_store.cc:276] 0628 07:52:41.533324 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.533723 (+   399us) service_pool.cc:225] Handling call
0628 07:52:43.319030 (+1785307us) ts_tablet_manager.cc:1145] Deleting tablet c4fcefaa9e954b52bbbee6e0a2f2f939
0628 07:52:43.362898 (+ 43868us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.376055 (+ 13157us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.423872 (+ 47817us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.483012 (+ 59140us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":22452,"spinlock_wait_cycles":694528,"tablet-delete.queue_time_us":1784766}
I20250628 07:52:43.484608   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]) successfully deleted
I20250628 07:52:43.485209   978 raft_consensus.cc:3058] T f845f32699044867af6f0f2b655cad8e P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.485527  2428 tablet_replica.cc:331] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:43.486243  2428 raft_consensus.cc:2241] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:43.487038  2428 raft_consensus.cc:2270] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.487816  2440 raft_consensus.cc:2747] T 29a77cd9dc244d239a518e004c49e3c8 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.490039  1229 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:43.492762   430 master-stress-test.cc:579] Killed test thread 1/10
I20250628 07:52:43.493029   430 master-stress-test.cc:577] Killing test thread 2/10
I20250628 07:52:43.493287   430 master-stress-test.cc:579] Killed test thread 2/10
I20250628 07:52:43.493484   430 master-stress-test.cc:577] Killing test thread 3/10
I20250628 07:52:43.493697   430 master-stress-test.cc:579] Killed test thread 3/10
I20250628 07:52:43.493911   430 master-stress-test.cc:577] Killing test thread 4/10
I20250628 07:52:43.494119   430 master-stress-test.cc:579] Killed test thread 4/10
I20250628 07:52:43.494303   430 master-stress-test.cc:577] Killing test thread 5/10
I20250628 07:52:43.494524   430 master-stress-test.cc:579] Killed test thread 5/10
I20250628 07:52:43.494707   430 master-stress-test.cc:577] Killing test thread 6/10
I20250628 07:52:43.494927   430 master-stress-test.cc:579] Killed test thread 6/10
I20250628 07:52:43.495133   430 master-stress-test.cc:577] Killing test thread 7/10
I20250628 07:52:43.495368  2237 raft_consensus.cc:2241] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.495848  2237 raft_consensus.cc:2270] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.496364  2428 ts_tablet_manager.cc:1905] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.502399  1229 tablet_bootstrap.cc:654] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.508352  1234 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:43.508744  1234 ts_tablet_manager.cc:1397] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.285s	user 0.009s	sys 0.000s
I20250628 07:52:43.510995  1234 raft_consensus.cc:357] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.511612  1234 raft_consensus.cc:383] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.511868  1234 raft_consensus.cc:738] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.512470  1234 consensus_queue.cc:260] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.530066  1230 tablet_bootstrap.cc:492] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:43.539202  1230 tablet_bootstrap.cc:654] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:43.541497  1062 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:43.542062  1062 leader_election.cc:343] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:43.542562  1062 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.542878  1062 leader_election.cc:304] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.543514  2606 raft_consensus.cc:2747] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:43.544340  1062 leader_election.cc:343] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.544668  1062 leader_election.cc:304] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.546151   956 tablet_service.cc:1468] Processing CreateTablet for tablet e7d4694dde5b4e47bfb981b46aaacbe5 (DEFAULT_TABLE table=default.Table_3b7ef98c85564dc9b40201e1713b7c55 [id=010e62c6a9a649d78b828d7a7dd2296f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.547084   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7d4694dde5b4e47bfb981b46aaacbe5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.547890  1229 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.548291  1229 ts_tablet_manager.cc:1397] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.075s	user 0.009s	sys 0.000s
I20250628 07:52:43.564652  2514 ts_tablet_manager.cc:1918] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:43.565160  2514 log.cc:1199] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/22b47b39f8eb45a0b5d2c5348bfbc55c
I20250628 07:52:43.565588  1229 raft_consensus.cc:357] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.566093  2514 ts_tablet_manager.cc:1939] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:43.566224  1229 raft_consensus.cc:383] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.566464  1229 raft_consensus.cc:738] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.567147  1229 consensus_queue.cc:260] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.568056  1099 tablet_service.cc:1468] Processing CreateTablet for tablet e7d4694dde5b4e47bfb981b46aaacbe5 (DEFAULT_TABLE table=default.Table_3b7ef98c85564dc9b40201e1713b7c55 [id=010e62c6a9a649d78b828d7a7dd2296f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.568440   958 tablet_service.cc:1468] Processing CreateTablet for tablet 987bf94636bd48b3a428ff7cc3b6bedc (DEFAULT_TABLE table=default.Table_753078c239dc43a6a5125732a2ff887c [id=eeec5abe55af475f8195a9ec1880ca62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.569144  1229 ts_tablet_manager.cc:1428] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.021s	user 0.007s	sys 0.000s
I20250628 07:52:43.569435   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 987bf94636bd48b3a428ff7cc3b6bedc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.570003  1229 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:43.570654  1234 ts_tablet_manager.cc:1428] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.062s	user 0.006s	sys 0.000s
I20250628 07:52:43.571497  1234 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:43.576371  2428 ts_tablet_manager.cc:1918] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:43.581722  2337 leader_election.cc:290] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:43.583601   792 tablet_service.cc:1468] Processing CreateTablet for tablet 987bf94636bd48b3a428ff7cc3b6bedc (DEFAULT_TABLE table=default.Table_753078c239dc43a6a5125732a2ff887c [id=eeec5abe55af475f8195a9ec1880ca62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.587160  2428 log.cc:1199] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/af13b56df157443c994f747e52b0f458
I20250628 07:52:43.587939  2428 ts_tablet_manager.cc:1939] T af13b56df157443c994f747e52b0f458 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:43.523947  2237 ts_tablet_manager.cc:1905] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.555135   789 tablet_service.cc:1468] Processing CreateTablet for tablet e7d4694dde5b4e47bfb981b46aaacbe5 (DEFAULT_TABLE table=default.Table_3b7ef98c85564dc9b40201e1713b7c55 [id=010e62c6a9a649d78b828d7a7dd2296f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.592806  1229 tablet_bootstrap.cc:654] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.593500  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 425) took 2060 ms. Trace:
I20250628 07:52:43.593815  2428 rpcz_store.cc:276] 0628 07:52:41.532976 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.537548 (+  4572us) service_pool.cc:225] Handling call
0628 07:52:43.485387 (+1947839us) ts_tablet_manager.cc:1145] Deleting tablet af13b56df157443c994f747e52b0f458
0628 07:52:43.485427 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.541693 (+ 56266us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.576175 (+ 34482us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.593443 (+ 17268us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1947292}
I20250628 07:52:43.595063   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:43.596091  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 427) took 2062 ms. Trace:
I20250628 07:52:43.597034   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 51a44ec161684f74b228830ff06f1957
I20250628 07:52:43.596380  2428 rpcz_store.cc:276] 0628 07:52:41.533617 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.538178 (+  4561us) service_pool.cc:225] Handling call
0628 07:52:43.595753 (+2057575us) ts_tablet_manager.cc:1145] Deleting tablet 06b9b63649384313948964b3c0dd11cf
0628 07:52:43.595810 (+    57us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.596043 (+   233us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2039881}
I20250628 07:52:43.598945   963 tablet_service.cc:1515] Processing DeleteTablet for tablet 51a44ec161684f74b228830ff06f1957 with delete_type TABLET_DATA_DELETED (replaced by tablet d96ec557ca9f4008846e8b99610174d5) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:43.601493   544 catalog_manager.cc:6652] ReplaceTablet: tablet 51a44ec161684f74b228830ff06f1957 deleted and replaced by tablet d96ec557ca9f4008846e8b99610174d5
W20250628 07:52:43.601797   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 06b9b63649384313948964b3c0dd11cf because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 06b9b63649384313948964b3c0dd11cf already in progress: creating tablet
I20250628 07:52:43.601794  1091 tablet_service.cc:1515] Processing DeleteTablet for tablet 51a44ec161684f74b228830ff06f1957 with delete_type TABLET_DATA_DELETED (replaced by tablet d96ec557ca9f4008846e8b99610174d5) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:43.600751  2337 raft_consensus.cc:491] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.602455  2337 raft_consensus.cc:513] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.604261  2337 leader_election.cc:290] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:43.604984  2428 tablet_replica.cc:331] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:43.605342  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:43.605830  2428 raft_consensus.cc:2241] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:43.605921  1122 raft_consensus.cc:2466] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:43.606637  2428 raft_consensus.cc:2270] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.606951   921 leader_election.cc:304] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:43.607995  1125 raft_consensus.cc:2466] T e52e15aefe194a61baa7e74616d9bfd3 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:43.608911   921 leader_election.cc:304] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:43.609885  2526 raft_consensus.cc:2802] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:43.610189  2526 raft_consensus.cc:491] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:43.611016   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.611239   790 tablet_service.cc:1515] Processing DeleteTablet for tablet 51a44ec161684f74b228830ff06f1957 with delete_type TABLET_DATA_DELETED (replaced by tablet d96ec557ca9f4008846e8b99610174d5) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:43.610540  2526 raft_consensus.cc:3058] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:43.612102   920 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.614377  2428 ts_tablet_manager.cc:1905] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.616714  2606 raft_consensus.cc:2747] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:43.616875  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 72551.5us
I20250628 07:52:43.618258   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.618443   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.622874  1091 tablet_service.cc:1468] Processing CreateTablet for tablet 987bf94636bd48b3a428ff7cc3b6bedc (DEFAULT_TABLE table=default.Table_753078c239dc43a6a5125732a2ff887c [id=eeec5abe55af475f8195a9ec1880ca62]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.625126   963 tablet_service.cc:1468] Processing CreateTablet for tablet f8e389edc2b645a6a5564422adf33641 (DEFAULT_TABLE table=default.Table_bb662e3441d74b638d7c33dc69755650 [id=6e9ea1b8ad2d4977a835f5b0c03d1dc8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.626071   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8e389edc2b645a6a5564422adf33641. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.626590   790 tablet_service.cc:1468] Processing CreateTablet for tablet f8e389edc2b645a6a5564422adf33641 (DEFAULT_TABLE table=default.Table_bb662e3441d74b638d7c33dc69755650 [id=6e9ea1b8ad2d4977a835f5b0c03d1dc8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.628111  2533 raft_consensus.cc:2802] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:43.628544  2533 raft_consensus.cc:695] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:43.629323  2533 consensus_queue.cc:237] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.630081   792 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 987bf94636bd48b3a428ff7cc3b6bedc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.632757  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 987bf94636bd48b3a428ff7cc3b6bedc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.633558   789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7d4694dde5b4e47bfb981b46aaacbe5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.637940   790 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8e389edc2b645a6a5564422adf33641. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.641577  1099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e7d4694dde5b4e47bfb981b46aaacbe5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.628679  2526 raft_consensus.cc:513] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.651227  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:43.651826  1121 raft_consensus.cc:3058] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:43.658586  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 397) took 1860 ms. Trace:
I20250628 07:52:43.658993  1121 raft_consensus.cc:2466] T fb3d9cc430354e54b82ed7844fd2181c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:43.665346   544 master_service.cc:948] ReplaceTablet: received request to replace tablet 670f91ee94f7460caa0790e232ac7fa8 from {username='slave'} at 127.0.0.1:47964
I20250628 07:52:43.668529  2526 leader_election.cc:290] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:43.672079   921 leader_election.cc:304] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:43.673707  2526 raft_consensus.cc:2802] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:43.674314  2526 raft_consensus.cc:695] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:43.675264  2526 consensus_queue.cc:237] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.659005  2514 rpcz_store.cc:276] 0628 07:52:41.798570 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.808670 (+ 10100us) service_pool.cc:225] Handling call
0628 07:52:43.132127 (+1323457us) ts_tablet_manager.cc:1145] Deleting tablet 22b47b39f8eb45a0b5d2c5348bfbc55c
0628 07:52:43.132170 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.404609 (+272439us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.564412 (+159803us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.658512 (+ 94100us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1322677}
I20250628 07:52:43.677728  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 399) took 1828 ms. Trace:
I20250628 07:52:43.677783   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (table default.Table_df287bcea5bd414ebe5aee1689733f8c [id=398556bc3bdc481cb6577321cbc31055]) successfully deleted
I20250628 07:52:43.679468  2514 rpcz_store.cc:276] 0628 07:52:41.849331 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.849448 (+   117us) service_pool.cc:225] Handling call
0628 07:52:43.677464 (+1828016us) ts_tablet_manager.cc:1145] Deleting tablet 63115b7be4a94b639c18601d438ed947
0628 07:52:43.677511 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.677678 (+   167us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1827397}
I20250628 07:52:43.679884  1230 tablet_bootstrap.cc:492] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:43.680413  1234 tablet_bootstrap.cc:654] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.680508  2514 tablet_replica.cc:331] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
W20250628 07:52:43.680727   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 63115b7be4a94b639c18601d438ed947 because the tablet was not found. No further retry: Not found: Tablet not found: 63115b7be4a94b639c18601d438ed947
I20250628 07:52:43.681398  2514 raft_consensus.cc:2241] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.681968  2514 raft_consensus.cc:2270] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.683923  2514 ts_tablet_manager.cc:1905] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.684242   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:43.680518  1230 ts_tablet_manager.cc:1397] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.151s	user 0.006s	sys 0.004s
W20250628 07:52:43.686653   920 leader_election.cc:343] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.687602  1230 raft_consensus.cc:357] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.688198  1230 raft_consensus.cc:383] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.688427  1230 raft_consensus.cc:738] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.689064  1230 consensus_queue.cc:260] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.693195  1229 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.693610  1229 ts_tablet_manager.cc:1397] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.124s	user 0.012s	sys 0.004s
I20250628 07:52:43.695700  1229 raft_consensus.cc:357] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.696367  1229 raft_consensus.cc:383] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.696537  1229 raft_consensus.cc:738] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.696952  1229 consensus_queue.cc:260] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.703949  1230 ts_tablet_manager.cc:1428] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.018s	user 0.001s	sys 0.005s
I20250628 07:52:43.704929  1230 tablet_bootstrap.cc:492] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:43.708392  1062 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.709873  1229 ts_tablet_manager.cc:1428] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.016s	user 0.005s	sys 0.000s
I20250628 07:52:43.710745  1229 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:43.729678  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 109840us
W20250628 07:52:43.730432  1062 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.730865  1062 leader_election.cc:304] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.731688  1229 tablet_bootstrap.cc:654] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.750329   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]). Expected version 2 got 0
I20250628 07:52:43.760249  1229 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.760620  1229 ts_tablet_manager.cc:1397] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.050s	user 0.005s	sys 0.006s
I20250628 07:52:43.761590  2237 ts_tablet_manager.cc:1918] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:43.761986  2237 log.cc:1199] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/2ae890710f7f44cf899dd19e96fa9181
I20250628 07:52:43.762595  2237 ts_tablet_manager.cc:1939] T 2ae890710f7f44cf899dd19e96fa9181 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:43.763051  1229 raft_consensus.cc:357] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.763381   791 tablet_service.cc:1468] Processing CreateTablet for tablet f2295f958cb34b658d2dfcf836d25e7f (DEFAULT_TABLE table=default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.763767  1229 raft_consensus.cc:383] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.764017  1229 raft_consensus.cc:738] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.764732  1229 consensus_queue.cc:260] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.766076  1234 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:43.770606  1098 tablet_service.cc:1468] Processing CreateTablet for tablet f8e389edc2b645a6a5564422adf33641 (DEFAULT_TABLE table=default.Table_bb662e3441d74b638d7c33dc69755650 [id=6e9ea1b8ad2d4977a835f5b0c03d1dc8]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.771596  1230 tablet_bootstrap.cc:654] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.772083  2514 ts_tablet_manager.cc:1918] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:43.772764  2514 log.cc:1199] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/f845f32699044867af6f0f2b655cad8e
I20250628 07:52:43.773509  2514 ts_tablet_manager.cc:1939] T f845f32699044867af6f0f2b655cad8e P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:43.778772  1229 ts_tablet_manager.cc:1428] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.018s	user 0.002s	sys 0.003s
I20250628 07:52:43.786942  2606 raft_consensus.cc:491] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.787369  2606 raft_consensus.cc:513] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.788996  2606 leader_election.cc:290] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.786536  1229 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:43.792838  1234 ts_tablet_manager.cc:1397] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.222s	user 0.010s	sys 0.000s
I20250628 07:52:43.793043  2606 raft_consensus.cc:2747] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:43.793746  2651 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
W20250628 07:52:43.793807  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 63405.2us
I20250628 07:52:43.794286  2651 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:43.797150  2651 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:43.797402   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:43.798666   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af28ae2139b2417381808cb4068ddf60" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:43.795562  1928 consensus_queue.cc:1035] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:43.801623   963 tablet_service.cc:1468] Processing CreateTablet for tablet f2295f958cb34b658d2dfcf836d25e7f (DEFAULT_TABLE table=default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.802629  1099 tablet_service.cc:1468] Processing CreateTablet for tablet f2295f958cb34b658d2dfcf836d25e7f (DEFAULT_TABLE table=default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.804262  1234 raft_consensus.cc:357] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.804893  1234 raft_consensus.cc:383] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.811053  1229 tablet_bootstrap.cc:654] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.814060  1098 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8e389edc2b645a6a5564422adf33641. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.821030  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 400) took 1971 ms. Trace:
I20250628 07:52:43.821452  2514 rpcz_store.cc:276] 0628 07:52:41.849649 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.850023 (+   374us) service_pool.cc:225] Handling call
0628 07:52:43.680352 (+1830329us) ts_tablet_manager.cc:1145] Deleting tablet f845f32699044867af6f0f2b655cad8e
0628 07:52:43.680390 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.701475 (+ 21085us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.746481 (+ 45006us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.820951 (+ 74470us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1829797}
I20250628 07:52:43.822604  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 401) took 1972 ms. Trace:
I20250628 07:52:43.822871  2514 rpcz_store.cc:276] 0628 07:52:41.849905 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.850345 (+   440us) service_pool.cc:225] Handling call
0628 07:52:43.822361 (+1972016us) ts_tablet_manager.cc:1145] Deleting tablet 5fa4299a9e0f4b409d04c8c5900901c0
0628 07:52:43.822410 (+    49us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.822548 (+   138us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1971478}
I20250628 07:52:43.823835  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 398) took 1974 ms. Trace:
I20250628 07:52:43.824097  2514 rpcz_store.cc:276] 0628 07:52:41.848985 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.901352 (+ 52367us) service_pool.cc:225] Handling call
0628 07:52:43.823612 (+1922260us) ts_tablet_manager.cc:1145] Deleting tablet e15d063a4bca40d480cba08733ec1ec3
0628 07:52:43.823652 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.823792 (+   140us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1921620}
I20250628 07:52:43.825874   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f2295f958cb34b658d2dfcf836d25e7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.826166  2514 tablet_replica.cc:331] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:43.827003  2514 raft_consensus.cc:2241] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:43.827445  2514 raft_consensus.cc:2270] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:43.828845  1099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f2295f958cb34b658d2dfcf836d25e7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.839337  2514 ts_tablet_manager.cc:1905] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:43.840368   956 tablet_service.cc:1468] Processing CreateTablet for tablet f06829296b994d4c9940bf7fe068ace7 (DEFAULT_TABLE table=default.Table_cc2df6969d8d43ba882f1832a7c4b528 [id=31b9ad86edb24981b8fbe350f2878d9a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.825182   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet f845f32699044867af6f0f2b655cad8e (table default.Table_92144695519f4cc9a21fedc487712d03 [id=77ccc4338ec448b6b07eace3de191874]) successfully deleted
W20250628 07:52:43.850126   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 5fa4299a9e0f4b409d04c8c5900901c0 because the tablet was not found. No further retry: Not found: Tablet not found: 5fa4299a9e0f4b409d04c8c5900901c0
W20250628 07:52:43.850728   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet e15d063a4bca40d480cba08733ec1ec3 because the tablet was not found. No further retry: Not found: Tablet not found: e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:43.852681   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:43.853909   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:43.854734   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 2 got 1
I20250628 07:52:43.857486  2606 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:43.858114  2606 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.860514  2606 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:43.867609  1230 tablet_bootstrap.cc:492] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:43.868021  1230 ts_tablet_manager.cc:1397] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.163s	user 0.006s	sys 0.018s
I20250628 07:52:43.870227  1230 raft_consensus.cc:357] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.870867  1230 raft_consensus.cc:383] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:43.871111  1230 raft_consensus.cc:738] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.873327  1230 consensus_queue.cc:260] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.876106  1230 ts_tablet_manager.cc:1428] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.001s
I20250628 07:52:43.879786  1091 tablet_service.cc:1468] Processing CreateTablet for tablet f06829296b994d4c9940bf7fe068ace7 (DEFAULT_TABLE table=default.Table_cc2df6969d8d43ba882f1832a7c4b528 [id=31b9ad86edb24981b8fbe350f2878d9a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.880966  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f06829296b994d4c9940bf7fe068ace7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.841559   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f06829296b994d4c9940bf7fe068ace7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.888132  1230 tablet_bootstrap.cc:492] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:43.854948  1229 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:43.890445  1229 ts_tablet_manager.cc:1397] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.111s	user 0.009s	sys 0.000s
I20250628 07:52:43.811861   802 tablet_service.cc:1468] Processing CreateTablet for tablet f06829296b994d4c9940bf7fe068ace7 (DEFAULT_TABLE table=default.Table_cc2df6969d8d43ba882f1832a7c4b528 [id=31b9ad86edb24981b8fbe350f2878d9a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.893064  2428 ts_tablet_manager.cc:1918] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:43.892807   958 tablet_service.cc:1468] Processing CreateTablet for tablet 20acb22449ac4203bbed31067df0c553 (DEFAULT_TABLE table=default.Table_e20c49ea9aae4980b56ee14957a8c928 [id=e7e6cd0579cf44d7877364fcb5c21e28]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:43.813226   757 consensus_peers.cc:489] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c4fcefaa9e954b52bbbee6e0a2f2f939. This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:43.893774   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 80694.8us
I20250628 07:52:43.840855   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af28ae2139b2417381808cb4068ddf60" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.844103   791 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f2295f958cb34b658d2dfcf836d25e7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.844784  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 413) took 3112 ms. Trace:
I20250628 07:52:43.895668   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f06829296b994d4c9940bf7fe068ace7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.895810  2237 rpcz_store.cc:276] 0628 07:52:40.732533 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.733543 (+  1010us) service_pool.cc:225] Handling call
0628 07:52:43.444831 (+2711288us) ts_tablet_manager.cc:1145] Deleting tablet 2ae890710f7f44cf899dd19e96fa9181
0628 07:52:43.444869 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.599291 (+154422us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.761348 (+162057us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.844628 (+ 83280us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2710789}
I20250628 07:52:43.906347  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 411) took 3177 ms. Trace:
I20250628 07:52:43.906674  2237 rpcz_store.cc:276] 0628 07:52:40.728541 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.728664 (+   123us) service_pool.cc:225] Handling call
0628 07:52:43.906085 (+3177421us) ts_tablet_manager.cc:1145] Deleting tablet 1c65fa809b17418c85d4d72ae1e0ee3a
0628 07:52:43.906134 (+    49us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.906294 (+   160us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3170956}
I20250628 07:52:43.907698  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 412) took 3176 ms. Trace:
I20250628 07:52:43.908203  2428 log.cc:1199] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/22b47b39f8eb45a0b5d2c5348bfbc55c
I20250628 07:52:43.908932  2428 ts_tablet_manager.cc:1939] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:43.894552   789 tablet_service.cc:1468] Processing CreateTablet for tablet 20acb22449ac4203bbed31067df0c553 (DEFAULT_TABLE table=default.Table_e20c49ea9aae4980b56ee14957a8c928 [id=e7e6cd0579cf44d7877364fcb5c21e28]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.916637  1230 tablet_bootstrap.cc:654] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:43.918895  1234 raft_consensus.cc:738] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:43.919754  1234 consensus_queue.cc:260] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:43.922215  1234 ts_tablet_manager.cc:1428] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.120s	user 0.003s	sys 0.001s
W20250628 07:52:43.934914   758 consensus_peers.cc:489] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f845f32699044867af6f0f2b655cad8e. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:43.936661  1928 consensus_queue.cc:1035] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:43.916961   789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 20acb22449ac4203bbed31067df0c553. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:43.941455  1094 tablet_service.cc:1468] Processing CreateTablet for tablet 4eac11942d104acab461d483252128c5 (DEFAULT_TABLE table=default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.939761  1091 tablet_service.cc:1468] Processing CreateTablet for tablet 20acb22449ac4203bbed31067df0c553 (DEFAULT_TABLE table=default.Table_e20c49ea9aae4980b56ee14957a8c928 [id=e7e6cd0579cf44d7877364fcb5c21e28]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.908004  2237 rpcz_store.cc:276] 0628 07:52:40.731339 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.735339 (+  4000us) service_pool.cc:225] Handling call
0628 07:52:43.907473 (+3172134us) ts_tablet_manager.cc:1145] Deleting tablet 46ff641001cd4b01a4c09c0d8b79ebf3
0628 07:52:43.907514 (+    41us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.907649 (+   135us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3171660}
I20250628 07:52:43.945006   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:43.947024  1234 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:43.881528   543 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 6ce66199d1614826bf618819f6928383
I20250628 07:52:43.950448   543 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 462) took 1469 ms. Trace:
I20250628 07:52:43.882061   544 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 670f91ee94f7460caa0790e232ac7fa8
I20250628 07:52:43.882305   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:43.954105   544 catalog_manager.cc:6652] ReplaceTablet: tablet 670f91ee94f7460caa0790e232ac7fa8 deleted and replaced by tablet 340b3ef1c06d48bc882e16e5765d4e22
I20250628 07:52:43.954196  2059 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:43.909976   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 2ae890710f7f44cf899dd19e96fa9181 (table default.Table_cfe73367b7384124927ff17e68820937 [id=a862da8979a544809d90f9b63f096feb]) successfully deleted
I20250628 07:52:43.955334  2651 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:52:43.955421   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 45475.1us
W20250628 07:52:43.957392   532 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 5475a89692eb450d82cde521ec8c9a73 (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5475a89692eb450d82cde521ec8c9a73 consensus state has no leader
W20250628 07:52:43.957854   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 1c65fa809b17418c85d4d72ae1e0ee3a because the tablet was not found. No further retry: Not found: Tablet not found: 1c65fa809b17418c85d4d72ae1e0ee3a
W20250628 07:52:43.958858   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 46ff641001cd4b01a4c09c0d8b79ebf3 because the tablet was not found. No further retry: Not found: Tablet not found: 46ff641001cd4b01a4c09c0d8b79ebf3
I20250628 07:52:43.950759   543 rpcz_store.cc:276] 0628 07:52:42.480764 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.537643 (+ 56879us) service_pool.cc:225] Handling call
0628 07:52:42.647526 (+109883us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:42.648655 (+  1129us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:42.648785 (+   130us) catalog_manager.cc:2799] Locking tablets
0628 07:52:43.540714 (+891929us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:43.751144 (+210430us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:43.881367 (+130223us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:43.881450 (+    83us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:43.950387 (+ 68937us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:43.588260 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:43.588783 (+   523us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:43.588804 (+    21us) write_op.cc:435] Acquired schema lock
0628 07:52:43.588816 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:43.589115 (+   299us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:43.589153 (+    38us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:43.589176 (+    23us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:43.737646 (+148470us) tablet.cc:699] Row locks acquired
0628 07:52:43.737665 (+    19us) write_op.cc:260] PREPARE: finished
0628 07:52:43.737772 (+   107us) write_op.cc:270] Start()
0628 07:52:43.738467 (+   695us) write_op.cc:276] Timestamp: P: 1751097163737579 usec, L: 0
0628 07:52:43.738488 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:43.738995 (+   507us) log.cc:844] Serialized 731 byte log entry
0628 07:52:43.749327 (+ 10332us) op_driver.cc:464] REPLICATION: finished
0628 07:52:43.749612 (+   285us) write_op.cc:301] APPLY: starting
0628 07:52:43.749668 (+    56us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:43.749771 (+   103us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:43.749850 (+    79us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:43.750078 (+   228us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:43.750202 (+   124us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:43.750240 (+    38us) write_op.cc:312] APPLY: finished
0628 07:52:43.750397 (+   157us) log.cc:844] Serialized 29 byte log entry
0628 07:52:43.750554 (+   157us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:43.750623 (+    69us) write_op.cc:454] Released schema lock
0628 07:52:43.750654 (+    31us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:43.750856 (+   202us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:43.750878 (+    22us) write_op.cc:454] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":143,"apply.run_cpu_time_us":1215,"apply.run_wall_time_us":1215,"num_ops":2,"prepare.queue_time_us":45975,"prepare.run_cpu_time_us":2088,"prepare.run_wall_time_us":150980,"raft.queue_time_us":892,"raft.run_cpu_time_us":588,"raft.run_wall_time_us":587,"replication_time_us":10748}]]}
I20250628 07:52:43.962102   430 master-stress-test.cc:579] Killed test thread 7/10
I20250628 07:52:43.962337   430 master-stress-test.cc:577] Killing test thread 8/10
I20250628 07:52:43.963835   803 tablet_service.cc:1515] Processing DeleteTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 with delete_type TABLET_DATA_DELETED (replaced by tablet 340b3ef1c06d48bc882e16e5765d4e22) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:43.965390   792 tablet_service.cc:1468] Processing CreateTablet for tablet 4eac11942d104acab461d483252128c5 (DEFAULT_TABLE table=default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.966095   790 tablet_service.cc:1515] Processing DeleteTablet for tablet 6ce66199d1614826bf618819f6928383 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:43.967362   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:43.967913   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (table default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960]). Expected version 1 got 0
I20250628 07:52:43.970932   542 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]). Expected version 1 got 0
I20250628 07:52:43.972994  1234 tablet_bootstrap.cc:654] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:43.979595  1062 leader_election.cc:343] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:43.989883   966 tablet_service.cc:1468] Processing CreateTablet for tablet 4eac11942d104acab461d483252128c5 (DEFAULT_TABLE table=default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:43.991185   957 tablet_service.cc:1515] Processing DeleteTablet for tablet 6ce66199d1614826bf618819f6928383 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:43.978906  2237 tablet_replica.cc:331] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:43.988374   545 catalog_manager.cc:5582] T 86591ef37c8547189ff8ef270e9bfa9a P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:43.998591   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:43.981730  2533 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:44.000626  1062 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:44.001189  1062 leader_election.cc:343] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.001508  1062 leader_election.cc:304] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:43.999799   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (table default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960]). Expected version 1 got 0
I20250628 07:52:44.002928   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]). Expected version 2 got 0
I20250628 07:52:43.842801  2337 consensus_queue.cc:1035] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.156s
I20250628 07:52:44.005335   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:43.999337   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.000919   954 tablet_service.cc:1515] Processing DeleteTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 with delete_type TABLET_DATA_DELETED (replaced by tablet 340b3ef1c06d48bc882e16e5765d4e22) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:44.004643  1090 tablet_service.cc:1515] Processing DeleteTablet for tablet 6ce66199d1614826bf618819f6928383 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:44.004526  1093 tablet_service.cc:1515] Processing DeleteTablet for tablet 670f91ee94f7460caa0790e232ac7fa8 with delete_type TABLET_DATA_DELETED (replaced by tablet 340b3ef1c06d48bc882e16e5765d4e22) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:44.017207  2606 raft_consensus.cc:2747] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.025556  1229 raft_consensus.cc:357] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.026194  1229 raft_consensus.cc:383] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.026443  1229 raft_consensus.cc:738] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.027096  1229 consensus_queue.cc:260] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.034979   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 20acb22449ac4203bbed31067df0c553. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.002602  2533 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.041446   545 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b]). Expected version 2 got 1
I20250628 07:52:44.042699  2533 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.043182   981 raft_consensus.cc:2466] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:44.043787  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.044369  1121 raft_consensus.cc:2466] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:44.048092  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 428) took 1164 ms. Trace:
I20250628 07:52:44.048399  2428 rpcz_store.cc:276] 0628 07:52:42.883805 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.884222 (+   417us) service_pool.cc:225] Handling call
0628 07:52:43.604807 (+720585us) ts_tablet_manager.cc:1145] Deleting tablet 22b47b39f8eb45a0b5d2c5348bfbc55c
0628 07:52:43.604845 (+    38us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:43.727722 (+122877us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:43.892849 (+165127us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.048020 (+155171us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":9492,"tablet-delete.queue_time_us":710435}
I20250628 07:52:44.050268  2533 consensus_queue.cc:1035] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:44.051561   921 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.052306  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 432) took 1139 ms. Trace:
I20250628 07:52:44.054801  2428 rpcz_store.cc:276] 0628 07:52:42.912584 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.912734 (+   150us) service_pool.cc:225] Handling call
0628 07:52:44.049435 (+1136701us) ts_tablet_manager.cc:1145] Deleting tablet ee03996105074308bda56d36ba3a1c5d
0628 07:52:44.052108 (+  2673us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.052259 (+   151us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1136171}
I20250628 07:52:44.055431  2337 raft_consensus.cc:2802] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.055685  2428 tablet_replica.cc:331] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:44.055754  2337 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.055781   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (table default.Table_df287bcea5bd414ebe5aee1689733f8c [id=398556bc3bdc481cb6577321cbc31055]) successfully deleted
I20250628 07:52:44.056097  2337 raft_consensus.cc:3058] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:44.056375   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet ee03996105074308bda56d36ba3a1c5d because the tablet was not found. No further retry: Not found: Tablet not found: ee03996105074308bda56d36ba3a1c5d
I20250628 07:52:44.059437   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:44.060449   920 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.062520   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4eac11942d104acab461d483252128c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.083138  1229 ts_tablet_manager.cc:1428] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.192s	user 0.008s	sys 0.000s
I20250628 07:52:44.083711  2514 ts_tablet_manager.cc:1918] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:44.084112  2514 log.cc:1199] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/1fe54aa8c2a04c238afb82ef9ff7fa81
I20250628 07:52:44.084745  2514 ts_tablet_manager.cc:1939] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:44.085755  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4eac11942d104acab461d483252128c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.089587  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 20acb22449ac4203bbed31067df0c553. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.090283  1230 tablet_bootstrap.cc:492] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:44.090678  1230 ts_tablet_manager.cc:1397] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.203s	user 0.017s	sys 0.013s
I20250628 07:52:44.097627  1230 raft_consensus.cc:357] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.098001  2428 raft_consensus.cc:2241] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.098249  1230 raft_consensus.cc:383] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.098500  1230 raft_consensus.cc:738] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.098695  2428 raft_consensus.cc:2270] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.106478  2428 ts_tablet_manager.cc:1905] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.109386  2237 raft_consensus.cc:2241] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.109925  2237 raft_consensus.cc:2270] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.111516  2237 ts_tablet_manager.cc:1905] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.111785  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 403) took 2143 ms. Trace:
I20250628 07:52:44.115520  2337 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.117159  1229 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:44.129257  2470 log.cc:927] Time spent T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276: Append to log took a long time: real 0.103s	user 0.007s	sys 0.009s
I20250628 07:52:44.134907   792 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4eac11942d104acab461d483252128c5. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:44.147040  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 145858us
I20250628 07:52:44.151222  1062 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:44.153543  2606 raft_consensus.cc:2802] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.153863  2606 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.154160  2606 raft_consensus.cc:3058] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.160418  2606 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.162967  2606 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.112128  2514 rpcz_store.cc:276] 0628 07:52:41.968524 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.970012 (+  1488us) service_pool.cc:225] Handling call
0628 07:52:43.826030 (+1856018us) ts_tablet_manager.cc:1145] Deleting tablet 1fe54aa8c2a04c238afb82ef9ff7fa81
0628 07:52:43.826076 (+    46us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.019220 (+193144us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.083491 (+ 64271us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.111722 (+ 28231us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1792682}
I20250628 07:52:44.165948   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 1fe54aa8c2a04c238afb82ef9ff7fa81 (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]) successfully deleted
I20250628 07:52:44.170017  1230 consensus_queue.cc:260] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.170542   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:44.171700  1928 raft_consensus.cc:491] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.172183  1928 raft_consensus.cc:513] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:44.173342  1062 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.170974  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 405) took 2161 ms. Trace:
I20250628 07:52:44.175501  1928 leader_election.cc:290] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.176326   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:44.176692  2606 raft_consensus.cc:491] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.177162  2606 raft_consensus.cc:513] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.179070  2606 leader_election.cc:290] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.186637   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "01b1767597424753a588090e9228d133" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.173756  2514 rpcz_store.cc:276] 0628 07:52:42.009811 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.010231 (+   420us) service_pool.cc:225] Handling call
0628 07:52:44.166598 (+2156367us) ts_tablet_manager.cc:1145] Deleting tablet 9ba2a10acdf0462bb05628351cdbbbe4
0628 07:52:44.170770 (+  4172us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.170924 (+   154us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2132913}
I20250628 07:52:44.196522  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 404) took 2188 ms. Trace:
I20250628 07:52:44.196808  2514 rpcz_store.cc:276] 0628 07:52:42.008595 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.013106 (+  4511us) service_pool.cc:225] Handling call
0628 07:52:44.190382 (+2177276us) ts_tablet_manager.cc:1145] Deleting tablet d87dd96d00964e12acd6e380a62c94d8
0628 07:52:44.196309 (+  5927us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.196475 (+   166us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2156494}
I20250628 07:52:44.197705  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.198171  1128 raft_consensus.cc:2466] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:44.198819  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 402) took 2232 ms. Trace:
I20250628 07:52:44.199105  2514 rpcz_store.cc:276] 0628 07:52:41.966538 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.971729 (+  5191us) service_pool.cc:225] Handling call
0628 07:52:44.197594 (+2225865us) ts_tablet_manager.cc:1145] Deleting tablet fa3ec14241e344029df3e39473181078
0628 07:52:44.198627 (+  1033us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.198768 (+   141us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2163388}
I20250628 07:52:44.200054  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 406) took 2190 ms. Trace:
I20250628 07:52:44.200332  2514 rpcz_store.cc:276] 0628 07:52:42.010105 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.021822 (+ 11717us) service_pool.cc:225] Handling call
0628 07:52:44.199838 (+2178016us) ts_tablet_manager.cc:1145] Deleting tablet 8155990eebd84fcf9ebc357e186d1bf2
0628 07:52:44.199874 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.200010 (+   136us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2165407}
W20250628 07:52:44.207722   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 174644us
I20250628 07:52:44.209439   758 leader_election.cc:304] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:44.210114  2441 raft_consensus.cc:2802] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.210433  2441 raft_consensus.cc:491] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.210752  2441 raft_consensus.cc:3058] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:44.212191  1062 leader_election.cc:343] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.172964  1230 ts_tablet_manager.cc:1428] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.082s	user 0.006s	sys 0.001s
I20250628 07:52:44.214020  2514 tablet_replica.cc:331] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:44.214715  2514 raft_consensus.cc:2241] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.215124  2514 raft_consensus.cc:2270] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.220276  2441 raft_consensus.cc:513] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.220784  1230 tablet_bootstrap.cc:492] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:44.221642   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "01b1767597424753a588090e9228d133" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:44.189913   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 9ba2a10acdf0462bb05628351cdbbbe4 because the tablet was not found. No further retry: Not found: Tablet not found: 9ba2a10acdf0462bb05628351cdbbbe4
W20250628 07:52:44.228420   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet d87dd96d00964e12acd6e380a62c94d8 because the tablet was not found. No further retry: Not found: Tablet not found: d87dd96d00964e12acd6e380a62c94d8
W20250628 07:52:44.229739   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet fa3ec14241e344029df3e39473181078 because the tablet was not found. No further retry: Not found: Tablet not found: fa3ec14241e344029df3e39473181078
W20250628 07:52:44.230291   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 8155990eebd84fcf9ebc357e186d1bf2 because the tablet was not found. No further retry: Not found: Tablet not found: 8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:44.222910   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:44.231853   978 raft_consensus.cc:2391] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 6a58514e01ea4989809bc2992fb2b806 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:44.233409  2428 ts_tablet_manager.cc:1918] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:44.233419  2514 ts_tablet_manager.cc:1905] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.237532  2428 log.cc:1199] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/fd67d7d9ba9446c19f1711fb5bd73ebd
I20250628 07:52:44.240408  2428 ts_tablet_manager.cc:1939] T fd67d7d9ba9446c19f1711fb5bd73ebd P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:44.242834  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 433) took 1330 ms. Trace:
I20250628 07:52:44.256750   539 catalog_manager.cc:5582] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:44.257829   539 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:44.258882  1230 tablet_bootstrap.cc:654] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.259640   539 catalog_manager.cc:5582] T 7d2a96449ed44d76a015e929d6a6edb0 P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:44.261222   539 catalog_manager.cc:5582] T 1ed24dbf9c234b43b352114d5219bbea P 6005c44dfe4848f18a304621cd49e934 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6005c44dfe4848f18a304621cd49e934 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "6005c44dfe4848f18a304621cd49e934" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:44.247403  2526 raft_consensus.cc:491] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.264183  2428 rpcz_store.cc:276] 0628 07:52:42.912852 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.912934 (+    82us) service_pool.cc:225] Handling call
0628 07:52:44.055541 (+1142607us) ts_tablet_manager.cc:1145] Deleting tablet fd67d7d9ba9446c19f1711fb5bd73ebd
0628 07:52:44.055586 (+    45us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.123318 (+ 67732us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.206687 (+ 83369us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.242775 (+ 36088us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":41457,"tablet-delete.queue_time_us":1142159}
I20250628 07:52:44.277669  2526 raft_consensus.cc:513] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.294498  2526 leader_election.cc:290] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:44.286108   921 consensus_peers.cc:489] T 1fe54aa8c2a04c238afb82ef9ff7fa81 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1fe54aa8c2a04c238afb82ef9ff7fa81. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:44.296648  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.297284  1126 raft_consensus.cc:2466] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:44.298220  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 434) took 1385 ms. Trace:
I20250628 07:52:44.286939  1234 tablet_bootstrap.cc:492] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:44.296634  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:44.298818  1234 ts_tablet_manager.cc:1397] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.352s	user 0.010s	sys 0.000s
I20250628 07:52:44.298957  1128 raft_consensus.cc:2391] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:44.303361   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet fd67d7d9ba9446c19f1711fb5bd73ebd (table default.Table_e26652b4c0454211b542434dc5992cd9 [id=ec582a4acf604066baa68b5762ff95bf]) successfully deleted
I20250628 07:52:44.306636   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (table default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960]). Expected version 1 got 0
W20250628 07:52:44.296663  1062 leader_election.cc:343] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.310338   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:44.310508  1062 leader_election.cc:304] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:44.312604  2606 raft_consensus.cc:2747] T 01b1767597424753a588090e9228d133 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.313709  1062 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:44.298275   921 leader_election.cc:304] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.293057  2337 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.315038  2428 rpcz_store.cc:276] 0628 07:52:42.913129 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.913767 (+   638us) service_pool.cc:225] Handling call
0628 07:52:44.297910 (+1384143us) ts_tablet_manager.cc:1145] Deleting tablet c3d1bc8483374a2a917451a43b3d42ed
0628 07:52:44.297954 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.298174 (+   220us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1383624}
I20250628 07:52:44.316396  2606 raft_consensus.cc:2747] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.305177   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.327123   546 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]). Expected version 1 got 0
I20250628 07:52:44.332310  2441 leader_election.cc:290] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:44.329668   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet c3d1bc8483374a2a917451a43b3d42ed because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c3d1bc8483374a2a917451a43b3d42ed already in progress: creating tablet
I20250628 07:52:44.319288   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:44.319902  2657 raft_consensus.cc:2802] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.328008  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 435) took 1414 ms. Trace:
I20250628 07:52:44.353477  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:44.353873  1229 tablet_bootstrap.cc:654] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.353997  1128 raft_consensus.cc:3058] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.353606  2428 rpcz_store.cc:276] 0628 07:52:42.913906 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.913984 (+    78us) service_pool.cc:225] Handling call
0628 07:52:44.327752 (+1413768us) ts_tablet_manager.cc:1145] Deleting tablet d87dd96d00964e12acd6e380a62c94d8
0628 07:52:44.327795 (+    43us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.327958 (+   163us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1413254}
I20250628 07:52:44.354889  2428 tablet_replica.cc:331] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
W20250628 07:52:44.354954   532 catalog_manager.cc:4895] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet d87dd96d00964e12acd6e380a62c94d8 because the tablet was not found. No further retry: Not found: Tablet not found: d87dd96d00964e12acd6e380a62c94d8
I20250628 07:52:44.355666  2657 raft_consensus.cc:491] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.356041  2657 raft_consensus.cc:3058] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.359941  1128 raft_consensus.cc:2466] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 1.
I20250628 07:52:44.361960   758 leader_election.cc:304] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:44.363154  1928 raft_consensus.cc:2802] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:44.363482  1928 raft_consensus.cc:695] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Becoming Leader. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Running, Role: LEADER
I20250628 07:52:44.364080  1928 consensus_queue.cc:237] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.364588  1230 tablet_bootstrap.cc:492] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:44.365325  2514 ts_tablet_manager.cc:1918] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:44.365676  2514 log.cc:1199] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/fd67d7d9ba9446c19f1711fb5bd73ebd
W20250628 07:52:44.369879  2665 log.cc:927] Time spent T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934: Append to log took a long time: real 0.051s	user 0.001s	sys 0.000s
I20250628 07:52:44.371141  1234 raft_consensus.cc:357] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.371752  1234 raft_consensus.cc:383] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.371981  1234 raft_consensus.cc:738] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:44.374173   757 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.375882  2441 consensus_queue.cc:1035] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:44.398831  2657 raft_consensus.cc:513] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:44.410820   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 128899us
I20250628 07:52:44.413149   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:44.414022   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.416548   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:44.416146  2657 leader_election.cc:290] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.417706  1229 tablet_bootstrap.cc:492] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:44.418131  1229 ts_tablet_manager.cc:1397] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.301s	user 0.007s	sys 0.003s
W20250628 07:52:44.419760   920 leader_election.cc:343] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.420140   920 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:44.420250  1229 raft_consensus.cc:357] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.420739  2669 raft_consensus.cc:2747] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.420842  1229 raft_consensus.cc:383] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20250628 07:52:44.421628   920 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:44.424224   920 leader_election.cc:343] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.421077  1229 raft_consensus.cc:738] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.428769  1234 consensus_queue.cc:260] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.429457  1229 consensus_queue.cc:260] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.430327  1234 ts_tablet_manager.cc:1428] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.131s	user 0.005s	sys 0.000s
I20250628 07:52:44.430816  1229 ts_tablet_manager.cc:1428] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.000s
I20250628 07:52:44.431207  1234 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:44.366268  2514 ts_tablet_manager.cc:1939] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:44.364987  1230 ts_tablet_manager.cc:1397] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.144s	user 0.011s	sys 0.000s
I20250628 07:52:44.436712  1234 tablet_bootstrap.cc:654] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.374712  2606 raft_consensus.cc:491] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.383942  2658 raft_consensus.cc:491] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.437330  2606 raft_consensus.cc:513] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.437393  2658 raft_consensus.cc:513] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.439042  2606 leader_election.cc:290] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.439170  2658 leader_election.cc:290] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.401479  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5f77a64b8e054f45acbad44d02af0e82" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:44.436841  1230 raft_consensus.cc:357] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.439710  1128 raft_consensus.cc:3058] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.439947  1230 raft_consensus.cc:383] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.440203  1230 raft_consensus.cc:738] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.440806  1230 consensus_queue.cc:260] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.442791  1230 ts_tablet_manager.cc:1428] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
I20250628 07:52:44.449736  1230 tablet_bootstrap.cc:492] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:44.447014  1128 raft_consensus.cc:2466] T 5f77a64b8e054f45acbad44d02af0e82 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:44.451982   921 leader_election.cc:304] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:44.452881  2669 raft_consensus.cc:2802] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:44.453313  2669 raft_consensus.cc:695] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:44.453975  2669 consensus_queue.cc:237] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.454718  1230 tablet_bootstrap.cc:654] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.463606   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "85c0ba070a8a40f2b32854ca24bc025c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.464076   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "924a153449b64d41b7db8c46afed93a9" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.464680  1229 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
W20250628 07:52:44.465229  1062 leader_election.cc:343] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:44.466451  1062 leader_election.cc:343] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.467849  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 407) took 2283 ms. Trace:
I20250628 07:52:44.468138  2514 rpcz_store.cc:276] 0628 07:52:42.184662 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.184790 (+   128us) service_pool.cc:225] Handling call
0628 07:52:44.213845 (+2029055us) ts_tablet_manager.cc:1145] Deleting tablet fd67d7d9ba9446c19f1711fb5bd73ebd
0628 07:52:44.213908 (+    63us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.333475 (+119567us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.365078 (+ 31603us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.467783 (+102705us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2015717}
I20250628 07:52:44.469249   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet fd67d7d9ba9446c19f1711fb5bd73ebd (table default.Table_e26652b4c0454211b542434dc5992cd9 [id=ec582a4acf604066baa68b5762ff95bf]) successfully deleted
I20250628 07:52:44.470297  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 408) took 2221 ms. Trace:
I20250628 07:52:44.470607  2514 rpcz_store.cc:276] 0628 07:52:42.248995 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.249170 (+   175us) service_pool.cc:225] Handling call
0628 07:52:44.470010 (+2220840us) ts_tablet_manager.cc:1145] Deleting tablet c3d1bc8483374a2a917451a43b3d42ed
0628 07:52:44.470047 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.470253 (+   206us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2220276}
I20250628 07:52:44.471285  1229 tablet_bootstrap.cc:654] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:44.471738   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c3d1bc8483374a2a917451a43b3d42ed because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c3d1bc8483374a2a917451a43b3d42ed already in progress: creating tablet
I20250628 07:52:44.492089  1230 tablet_bootstrap.cc:492] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:44.496418  1229 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:44.496807  1229 ts_tablet_manager.cc:1397] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.000s
I20250628 07:52:44.496740  2657 raft_consensus.cc:491] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.497371  2657 raft_consensus.cc:513] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.499202  2657 leader_election.cc:290] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.492559  1230 ts_tablet_manager.cc:1397] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.043s	user 0.008s	sys 0.000s
I20250628 07:52:44.500512  1229 raft_consensus.cc:357] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.501233  1229 raft_consensus.cc:383] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.501461  1229 raft_consensus.cc:738] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.501981  1229 consensus_queue.cc:260] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.502173  1230 raft_consensus.cc:357] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.503234  1230 raft_consensus.cc:383] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.503343  1229 ts_tablet_manager.cc:1428] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.006s	user 0.002s	sys 0.000s
I20250628 07:52:44.503545  1230 raft_consensus.cc:738] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.515529  1229 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:44.518126  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.519058  1125 raft_consensus.cc:2466] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:44.520908   921 leader_election.cc:304] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.522656  2533 raft_consensus.cc:2802] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.522975  2533 raft_consensus.cc:491] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.523268  2533 raft_consensus.cc:3058] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.530504  2533 raft_consensus.cc:513] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.531183  1229 tablet_bootstrap.cc:654] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.533465  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:44.533972  1125 raft_consensus.cc:3058] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.540285  1125 raft_consensus.cc:2466] T 3519e872e8504a4fb962814e3438bc01 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:44.541193   921 leader_election.cc:304] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.542114  2657 raft_consensus.cc:2802] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:44.542618  2657 raft_consensus.cc:695] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:44.543422  2657 consensus_queue.cc:237] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.545868  2668 consensus_queue.cc:1035] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:44.547235  1229 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:44.547639  1229 ts_tablet_manager.cc:1397] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.000s
I20250628 07:52:44.548014  2533 leader_election.cc:290] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.548346  2658 raft_consensus.cc:491] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.548780  2658 raft_consensus.cc:513] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.550308  1229 raft_consensus.cc:357] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.550843  2658 leader_election.cc:290] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.550904  1229 raft_consensus.cc:383] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.551018  2514 tablet_replica.cc:331] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:44.551146  1229 raft_consensus.cc:738] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.551743   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a168c27bfea94f9bba06c3a88329a892" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.551728  1229 consensus_queue.cc:260] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.552372  2514 raft_consensus.cc:2241] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.552786  2514 raft_consensus.cc:2270] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:44.553113  1062 leader_election.cc:343] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.553272  1229 ts_tablet_manager.cc:1428] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.005s	user 0.007s	sys 0.000s
I20250628 07:52:44.554035  1229 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:44.560247  2514 ts_tablet_manager.cc:1905] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.567636  2428 raft_consensus.cc:2241] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.568084  2428 raft_consensus.cc:2270] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.569751  2428 ts_tablet_manager.cc:1905] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.588116  2428 ts_tablet_manager.cc:1918] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:44.588461  2428 log.cc:1199] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:44.591989  1230 consensus_queue.cc:260] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.594493  1230 ts_tablet_manager.cc:1428] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.095s	user 0.005s	sys 0.000s
I20250628 07:52:44.602926  1229 tablet_bootstrap.cc:654] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.613744  2428 ts_tablet_manager.cc:1939] T 8155990eebd84fcf9ebc357e186d1bf2 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
I20250628 07:52:44.615845  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 436) took 1701 ms. Trace:
I20250628 07:52:44.616134  2428 rpcz_store.cc:276] 0628 07:52:42.914183 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.914591 (+   408us) service_pool.cc:225] Handling call
0628 07:52:44.354742 (+1440151us) ts_tablet_manager.cc:1145] Deleting tablet 8155990eebd84fcf9ebc357e186d1bf2
0628 07:52:44.354781 (+    39us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.582065 (+227284us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.587929 (+  5864us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.615794 (+ 27865us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":211926,"tablet-delete.queue_time_us":1439648}
I20250628 07:52:44.617300   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 8155990eebd84fcf9ebc357e186d1bf2 (table default.Table_39092c471b2443d183b7229a1d625b66 [id=ab01e12b48e047828205036a96ce6762]) successfully deleted
I20250628 07:52:44.622634  2514 ts_tablet_manager.cc:1918] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:44.622995  2514 log.cc:1199] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/29a77cd9dc244d239a518e004c49e3c8
I20250628 07:52:44.623625  2514 ts_tablet_manager.cc:1939] T 29a77cd9dc244d239a518e004c49e3c8 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:44.625680  1230 tablet_bootstrap.cc:492] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:44.625604  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 437) took 1711 ms. Trace:
I20250628 07:52:44.625885  2428 rpcz_store.cc:276] 0628 07:52:42.914462 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.936124 (+ 21662us) service_pool.cc:225] Handling call
0628 07:52:44.625298 (+1689174us) ts_tablet_manager.cc:1145] Deleting tablet 88d4a84c8ed7475c9bf043aaba8ad624
0628 07:52:44.625334 (+    36us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.625559 (+   225us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1688686}
W20250628 07:52:44.626946   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 88d4a84c8ed7475c9bf043aaba8ad624 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 88d4a84c8ed7475c9bf043aaba8ad624 already in progress: creating tablet
I20250628 07:52:44.631023  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 409) took 2107 ms. Trace:
I20250628 07:52:44.631337  2514 rpcz_store.cc:276] 0628 07:52:42.523346 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.523439 (+    93us) service_pool.cc:225] Handling call
0628 07:52:44.490129 (+1966690us) ts_tablet_manager.cc:1145] Deleting tablet 29a77cd9dc244d239a518e004c49e3c8
0628 07:52:44.516532 (+ 26403us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.600879 (+ 84347us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.622418 (+ 21539us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.630972 (+  8554us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1966149}
I20250628 07:52:44.632681   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 29a77cd9dc244d239a518e004c49e3c8 (table default.Table_026074aa331e455ebaaf270237f94590 [id=a4c74a0d7c804bbbb875dd4ee2607710]) successfully deleted
I20250628 07:52:44.637121  2514 tablet_replica.cc:331] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:44.639245  1229 tablet_bootstrap.cc:492] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:44.639627  1229 ts_tablet_manager.cc:1397] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.086s	user 0.007s	sys 0.000s
I20250628 07:52:44.644378  1229 raft_consensus.cc:357] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.644956  1229 raft_consensus.cc:383] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.645190  1229 raft_consensus.cc:738] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.645725  1229 consensus_queue.cc:260] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.648305  2428 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:35848 (request call id 440) took 1049 ms. Trace:
I20250628 07:52:44.648742  2428 rpcz_store.cc:276] 0628 07:52:43.598696 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.598820 (+   124us) service_pool.cc:225] Handling call
0628 07:52:44.647871 (+1049051us) ts_tablet_manager.cc:1145] Deleting tablet 51a44ec161684f74b228830ff06f1957
0628 07:52:44.647930 (+    59us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.648235 (+   305us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1046594}
W20250628 07:52:44.649865   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 51a44ec161684f74b228830ff06f1957 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 51a44ec161684f74b228830ff06f1957 already in progress: creating tablet
W20250628 07:52:44.650559   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 6ce66199d1614826bf618819f6928383 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 6ce66199d1614826bf618819f6928383 already in progress: creating tablet
W20250628 07:52:44.651117   532 catalog_manager.cc:4908] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): delete failed for tablet 670f91ee94f7460caa0790e232ac7fa8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 670f91ee94f7460caa0790e232ac7fa8 already in progress: creating tablet
I20250628 07:52:44.654114  1229 ts_tablet_manager.cc:1428] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.014s	user 0.007s	sys 0.000s
I20250628 07:52:44.654947  1229 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:44.684132   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a168c27bfea94f9bba06c3a88329a892" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:44.685781  1062 leader_election.cc:343] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.686159  1062 leader_election.cc:304] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:44.688621   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "924a153449b64d41b7db8c46afed93a9" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.695248   958 tablet_service.cc:1468] Processing CreateTablet for tablet 8e843597f12144799e7e13a7a4b1cc56 (DEFAULT_TABLE table=default.Table_3f9fa575c08344b9965c9fb9bccc8c85 [id=a8e8d7b379564c7699f75b69b507bb73]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.696328   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e843597f12144799e7e13a7a4b1cc56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.688676   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "85c0ba070a8a40f2b32854ca24bc025c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.710748  1091 tablet_service.cc:1468] Processing CreateTablet for tablet 8e843597f12144799e7e13a7a4b1cc56 (DEFAULT_TABLE table=default.Table_3f9fa575c08344b9965c9fb9bccc8c85 [id=a8e8d7b379564c7699f75b69b507bb73]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.711078  2514 raft_consensus.cc:2241] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:44.711207   956 tablet_service.cc:1468] Processing CreateTablet for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd (DEFAULT_TABLE table=default.Table_c35cbc8840ec40728f62ffc4fb20ce69 [id=41902d471a304c5d985c48efbe81acd0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.711599  2606 raft_consensus.cc:2747] T a168c27bfea94f9bba06c3a88329a892 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.711804  2514 raft_consensus.cc:2270] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.712154   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:44.720605  1062 leader_election.cc:343] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.720992  1062 leader_election.cc:304] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:44.725325  1062 leader_election.cc:343] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.725730  1062 leader_election.cc:304] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:44.735525   966 tablet_service.cc:1468] Processing CreateTablet for tablet ddac805491f64b1a95c7cc7d3dba8973 (DEFAULT_TABLE table=default.Table_4bed6916b6b14023b7099f15be8c1edc [id=8d3271dc7fa24639bdc1bf5d4104a459]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.736413   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ddac805491f64b1a95c7cc7d3dba8973. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.752692  2514 ts_tablet_manager.cc:1905] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:44.728731  2237 ts_tablet_manager.cc:1918] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:44.757830  2237 log.cc:1199] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/fa3ec14241e344029df3e39473181078
I20250628 07:52:44.757784  1094 tablet_service.cc:1468] Processing CreateTablet for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd (DEFAULT_TABLE table=default.Table_c35cbc8840ec40728f62ffc4fb20ce69 [id=41902d471a304c5d985c48efbe81acd0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.758891  2237 ts_tablet_manager.cc:1939] T fa3ec14241e344029df3e39473181078 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:44.758972  1099 tablet_service.cc:1468] Processing CreateTablet for tablet ddac805491f64b1a95c7cc7d3dba8973 (DEFAULT_TABLE table=default.Table_4bed6916b6b14023b7099f15be8c1edc [id=8d3271dc7fa24639bdc1bf5d4104a459]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.759980  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e843597f12144799e7e13a7a4b1cc56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.762399  2668 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.762828  2668 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.765209  2606 raft_consensus.cc:2747] T 85c0ba070a8a40f2b32854ca24bc025c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.765269  2668 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.751783  2658 raft_consensus.cc:2747] T 924a153449b64d41b7db8c46afed93a9 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:44.768599  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.769134  1125 raft_consensus.cc:2466] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:44.773753  1099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ddac805491f64b1a95c7cc7d3dba8973. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.773574   963 tablet_service.cc:1468] Processing CreateTablet for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e (DEFAULT_TABLE table=default.Table_743f866ef4ae435180af5ecffcb04bc2 [id=b3a09493549c479499d0eb68d3841bc7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.774480   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.777313  2688 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.778108  2688 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.780349  2688 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.781670  1094 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.786972   921 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.789273  2669 raft_consensus.cc:2802] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.789570  2669 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.789862  2669 raft_consensus.cc:3058] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.790109  2657 consensus_queue.cc:1035] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:44.796564  2669 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.798550  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 416) took 3904 ms. Trace:
I20250628 07:52:44.798926  2237 rpcz_store.cc:276] 0628 07:52:40.894167 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.894596 (+   429us) service_pool.cc:225] Handling call
0628 07:52:43.945904 (+3051308us) ts_tablet_manager.cc:1145] Deleting tablet fa3ec14241e344029df3e39473181078
0628 07:52:43.978758 (+ 32854us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.449195 (+470437us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.728291 (+279096us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.798448 (+ 70157us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":36516,"tablet-delete.queue_time_us":3050759}
I20250628 07:52:44.800379  2237 tablet_replica.cc:331] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
W20250628 07:52:44.801749  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 72220.7us
I20250628 07:52:44.803279   789 tablet_service.cc:1468] Processing CreateTablet for tablet 8e843597f12144799e7e13a7a4b1cc56 (DEFAULT_TABLE table=default.Table_3f9fa575c08344b9965c9fb9bccc8c85 [id=a8e8d7b379564c7699f75b69b507bb73]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.804612   789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8e843597f12144799e7e13a7a4b1cc56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.805133  1090 tablet_service.cc:1468] Processing CreateTablet for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e (DEFAULT_TABLE table=default.Table_743f866ef4ae435180af5ecffcb04bc2 [id=b3a09493549c479499d0eb68d3841bc7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.806171  1090 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.834630  1093 tablet_service.cc:1468] Processing CreateTablet for tablet 12d856ea75ef4f558efc58afd73339f4 (DEFAULT_TABLE table=default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 [id=6de11b06df364014af9b55fa276f3a7a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.835692  1093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 12d856ea75ef4f558efc58afd73339f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.822096   954 tablet_service.cc:1468] Processing CreateTablet for tablet 12d856ea75ef4f558efc58afd73339f4 (DEFAULT_TABLE table=default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 [id=6de11b06df364014af9b55fa276f3a7a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.837116   954 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 12d856ea75ef4f558efc58afd73339f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.846608  1126 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:44.847152  1126 raft_consensus.cc:2466] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6005c44dfe4848f18a304621cd49e934 in term 0.
I20250628 07:52:44.847234  1234 tablet_bootstrap.cc:492] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:44.847678  1234 ts_tablet_manager.cc:1397] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.417s	user 0.006s	sys 0.011s
I20250628 07:52:44.850054   792 tablet_service.cc:1468] Processing CreateTablet for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd (DEFAULT_TABLE table=default.Table_c35cbc8840ec40728f62ffc4fb20ce69 [id=41902d471a304c5d985c48efbe81acd0]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.851456   792 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.858920   802 tablet_service.cc:1468] Processing CreateTablet for tablet ddac805491f64b1a95c7cc7d3dba8973 (DEFAULT_TABLE table=default.Table_4bed6916b6b14023b7099f15be8c1edc [id=8d3271dc7fa24639bdc1bf5d4104a459]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.859220   791 tablet_service.cc:1468] Processing CreateTablet for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e (DEFAULT_TABLE table=default.Table_743f866ef4ae435180af5ecffcb04bc2 [id=b3a09493549c479499d0eb68d3841bc7]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.859923   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ddac805491f64b1a95c7cc7d3dba8973. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.860548   791 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.863476   790 tablet_service.cc:1468] Processing CreateTablet for tablet 12d856ea75ef4f558efc58afd73339f4 (DEFAULT_TABLE table=default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 [id=6de11b06df364014af9b55fa276f3a7a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.864542   790 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 12d856ea75ef4f558efc58afd73339f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.865852  1128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:44.866336  1128 raft_consensus.cc:3058] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.872349  1128 raft_consensus.cc:2466] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:44.873304   921 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:44.874063  2668 raft_consensus.cc:2802] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:44.874467  2668 raft_consensus.cc:695] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:44.875200  2668 consensus_queue.cc:237] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.876214  1229 tablet_bootstrap.cc:654] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.878587   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.883599  2237 raft_consensus.cc:2241] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:44.887566   966 tablet_service.cc:1468] Processing CreateTablet for tablet 174cab60958f41759967ffe9dda27991 (DEFAULT_TABLE table=default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 [id=2609d3d6796f4fa0b9cb750fa925da21]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.888453   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 174cab60958f41759967ffe9dda27991. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.889880   803 tablet_service.cc:1468] Processing CreateTablet for tablet 174cab60958f41759967ffe9dda27991 (DEFAULT_TABLE table=default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 [id=2609d3d6796f4fa0b9cb750fa925da21]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.890893   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 174cab60958f41759967ffe9dda27991. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.892854   958 tablet_service.cc:1468] Processing CreateTablet for tablet 24625050151f4b50aa8349c35184570a (DEFAULT_TABLE table=default.Table_3cf97e46be3049c0b1faa4e17cfa70eb [id=d8e81f13937f4fde8192b2293190c950]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.895273   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 24625050151f4b50aa8349c35184570a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.894145   963 tablet_service.cc:1468] Processing CreateTablet for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7 (DEFAULT_TABLE table=default.Table_ecc8fdb9f92341d38931d4b776b8114a [id=97a9ff9b1c5e4f808220992750b079a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.896458   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.899957   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]). Expected version 2 got 0
I20250628 07:52:44.902993   957 tablet_service.cc:1468] Processing CreateTablet for tablet 7ae51bcf0bd94770a818b504749184fc (DEFAULT_TABLE table=default.Table_061d0f52cd51486a88f1e59480dffc7f [id=da004477b85348e69492c13b6404eed4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.908020   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.908452   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:44.910007   818 raft_consensus.cc:3058] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.912428  1095 tablet_service.cc:1468] Processing CreateTablet for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7 (DEFAULT_TABLE table=default.Table_ecc8fdb9f92341d38931d4b776b8114a [id=97a9ff9b1c5e4f808220992750b079a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.910096  2658 raft_consensus.cc:491] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.913561  1092 tablet_service.cc:1468] Processing CreateTablet for tablet 174cab60958f41759967ffe9dda27991 (DEFAULT_TABLE table=default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 [id=2609d3d6796f4fa0b9cb750fa925da21]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.914228   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ae51bcf0bd94770a818b504749184fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.914026  2658 raft_consensus.cc:513] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.914671  1102 tablet_service.cc:1468] Processing CreateTablet for tablet 7ae51bcf0bd94770a818b504749184fc (DEFAULT_TABLE table=default.Table_061d0f52cd51486a88f1e59480dffc7f [id=da004477b85348e69492c13b6404eed4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.915707  2658 leader_election.cc:290] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.913583  1098 tablet_service.cc:1468] Processing CreateTablet for tablet 24625050151f4b50aa8349c35184570a (DEFAULT_TABLE table=default.Table_3cf97e46be3049c0b1faa4e17cfa70eb [id=d8e81f13937f4fde8192b2293190c950]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.917140  1096 tablet_service.cc:1468] Processing CreateTablet for tablet c6b42038b74a450796b43fa89d462017 (DEFAULT_TABLE table=default.Table_992ef1fbcdb840c382cc60a218166855 [id=868a11eb38d6484b98187acb9813872f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.922003  1106 tablet_service.cc:1468] Processing CreateTablet for tablet 9130fcc0c37f4d8e8852719fbafc3695 (DEFAULT_TABLE table=default.Table_fe7551bb42dd452ab32b051075e3398b [id=6ad3b42bbcbd4d5e894480654bf6bebe]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.924429  2514 ts_tablet_manager.cc:1918] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:52:44.924772  2514 log.cc:1199] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/a8518c82c188484c8554d3115637f5be
I20250628 07:52:44.924590  1090 tablet_service.cc:1468] Processing CreateTablet for tablet d200ea874eca4774a709c592a871e5ba (DEFAULT_TABLE table=default.Table_b6f2044e4f634f6182beb27ccdc55512 [id=11375d9db9b940f2bc7bfcb3c9904cac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.925408  2514 ts_tablet_manager.cc:1939] T a8518c82c188484c8554d3115637f5be P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:44.925436  2237 raft_consensus.cc:2270] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:44.926505  1234 raft_consensus.cc:357] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.927273  1234 raft_consensus.cc:383] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:44.927644  2237 ts_tablet_manager.cc:1905] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:44.927817   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 49448.1us
I20250628 07:52:44.930155   758 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806; no voters: 
I20250628 07:52:44.931555  2683 raft_consensus.cc:2802] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:44.931860  2683 raft_consensus.cc:491] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:44.932132  2683 raft_consensus.cc:3058] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:44.935544  1230 tablet_bootstrap.cc:654] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:44.938510   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:44.927675  1234 raft_consensus.cc:738] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:44.941920  1234 consensus_queue.cc:260] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.945397   540 catalog_manager.cc:5582] T 8226c926201040c993804d2ae8450202 P 6a58514e01ea4989809bc2992fb2b806 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } } }
I20250628 07:52:44.946740   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.949580   956 tablet_service.cc:1468] Processing CreateTablet for tablet c6b42038b74a450796b43fa89d462017 (DEFAULT_TABLE table=default.Table_992ef1fbcdb840c382cc60a218166855 [id=868a11eb38d6484b98187acb9813872f]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:44.949761  1062 leader_election.cc:343] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:44.950654   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6b42038b74a450796b43fa89d462017. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.953238  2658 raft_consensus.cc:491] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:44.953197   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.953641  2658 raft_consensus.cc:513] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:44.955360  2658 leader_election.cc:290] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:44.955402   789 tablet_service.cc:1468] Processing CreateTablet for tablet 24625050151f4b50aa8349c35184570a (DEFAULT_TABLE table=default.Table_3cf97e46be3049c0b1faa4e17cfa70eb [id=d8e81f13937f4fde8192b2293190c950]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.956218   820 raft_consensus.cc:3058] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:44.958037   757 consensus_peers.cc:489] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 -> Peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Couldn't send request to peer 82f43cdfab544b17a6a71033983e65d4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8155990eebd84fcf9ebc357e186d1bf2. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:44.960640  1097 tablet_service.cc:1468] Processing CreateTablet for tablet 48b376ff0fbc47b086870e1503bd2d20 (DEFAULT_TABLE table=default.Table_0f6f46bde0ed463eb196dfdf0f049317 [id=6ceec3d9bfe142268afe00a134a83201]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.892378  2669 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:44.961653   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.962172  1234 ts_tablet_manager.cc:1428] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.114s	user 0.005s	sys 0.004s
I20250628 07:52:44.965355   981 raft_consensus.cc:2466] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:44.969679  1108 tablet_service.cc:1468] Processing CreateTablet for tablet 71d772c683f94e79bd14bfc139357844 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.976297   953 tablet_service.cc:1468] Processing CreateTablet for tablet 9130fcc0c37f4d8e8852719fbafc3695 (DEFAULT_TABLE table=default.Table_fe7551bb42dd452ab32b051075e3398b [id=6ad3b42bbcbd4d5e894480654bf6bebe]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.977429   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fc4c7770fb564ccd95b3c789b179eecf" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:44.977452   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fc4c7770fb564ccd95b3c789b179eecf" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:44.977694   953 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9130fcc0c37f4d8e8852719fbafc3695. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.978693   963 tablet_service.cc:1468] Processing CreateTablet for tablet 48b376ff0fbc47b086870e1503bd2d20 (DEFAULT_TABLE table=default.Table_0f6f46bde0ed463eb196dfdf0f049317 [id=6ceec3d9bfe142268afe00a134a83201]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.979564   963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48b376ff0fbc47b086870e1503bd2d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.983729   540 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]). Expected version 1 got 0
I20250628 07:52:44.985940   958 tablet_service.cc:1468] Processing CreateTablet for tablet 71d772c683f94e79bd14bfc139357844 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.987540   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71d772c683f94e79bd14bfc139357844. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:44.989049  2237 ts_tablet_manager.cc:1918] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:44.989493  2237 log.cc:1199] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/63115b7be4a94b639c18601d438ed947
I20250628 07:52:44.990356  2237 ts_tablet_manager.cc:1939] T 63115b7be4a94b639c18601d438ed947 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:44.994621   951 tablet_service.cc:1468] Processing CreateTablet for tablet d200ea874eca4774a709c592a871e5ba (DEFAULT_TABLE table=default.Table_b6f2044e4f634f6182beb27ccdc55512 [id=11375d9db9b940f2bc7bfcb3c9904cac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:44.995640   951 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d200ea874eca4774a709c592a871e5ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.014201  1092 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 174cab60958f41759967ffe9dda27991. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.018462  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 410) took 2494 ms. Trace:
I20250628 07:52:45.018800  2514 rpcz_store.cc:276] 0628 07:52:42.523708 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.524064 (+   356us) service_pool.cc:225] Handling call
0628 07:52:44.636922 (+2112858us) ts_tablet_manager.cc:1145] Deleting tablet a8518c82c188484c8554d3115637f5be
0628 07:52:44.636977 (+    55us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.821476 (+184499us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.924241 (+102765us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.018406 (+ 94165us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":73358,"tablet-delete.queue_time_us":2112338}
I20250628 07:52:45.019840  2514 tablet_replica.cc:331] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:45.020754   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet a8518c82c188484c8554d3115637f5be (table default.Table_32928f5c3c044deaa68ad9b7971a3aaf [id=65bbed71d6394e34a7a2f3f3f5fa069d]) successfully deleted
I20250628 07:52:45.029500  1102 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ae51bcf0bd94770a818b504749184fc. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:45.033171   757 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 73817.5us
I20250628 07:52:45.033574  1096 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6b42038b74a450796b43fa89d462017. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.037846  1090 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d200ea874eca4774a709c592a871e5ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.038393  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71d772c683f94e79bd14bfc139357844. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.046460   543 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:44.939376  2683 raft_consensus.cc:513] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.047816   817 raft_consensus.cc:2391] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 82f43cdfab544b17a6a71033983e65d4 in current term 1: Already voted for candidate 6005c44dfe4848f18a304621cd49e934 in this term.
I20250628 07:52:45.049549  2683 leader_election.cc:290] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.053972  1106 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9130fcc0c37f4d8e8852719fbafc3695. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.057590  1098 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 24625050151f4b50aa8349c35184570a. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:45.060914   920 leader_election.cc:343] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.061690  1097 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48b376ff0fbc47b086870e1503bd2d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.065757  1095 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.065683   820 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:45.069854  2514 raft_consensus.cc:2241] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:45.070389  2514 raft_consensus.cc:2270] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:45.075219   981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:45.077633  2514 ts_tablet_manager.cc:1905] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:45.104512   802 tablet_service.cc:1468] Processing CreateTablet for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7 (DEFAULT_TABLE table=default.Table_ecc8fdb9f92341d38931d4b776b8114a [id=97a9ff9b1c5e4f808220992750b079a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.106163   799 tablet_service.cc:1468] Processing CreateTablet for tablet 7ae51bcf0bd94770a818b504749184fc (DEFAULT_TABLE table=default.Table_061d0f52cd51486a88f1e59480dffc7f [id=da004477b85348e69492c13b6404eed4]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.107074  2658 raft_consensus.cc:491] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.107609  2658 raft_consensus.cc:513] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.109452  2658 leader_election.cc:290] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:45.110558   802 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.110602  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 414) took 4217 ms. Trace:
I20250628 07:52:45.110926  2237 rpcz_store.cc:276] 0628 07:52:40.893614 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:40.898565 (+  4951us) service_pool.cc:225] Handling call
0628 07:52:44.800145 (+3901580us) ts_tablet_manager.cc:1145] Deleting tablet 63115b7be4a94b639c18601d438ed947
0628 07:52:44.800236 (+    91us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:44.937580 (+137344us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:44.975122 (+ 37542us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.110537 (+135415us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":40872,"tablet-delete.queue_time_us":3897849}
I20250628 07:52:45.111994  2237 tablet_replica.cc:331] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:45.112514  1234 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:45.113370   982 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "878b25ab3282423fab41014a8ea8949c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.113804   799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7ae51bcf0bd94770a818b504749184fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.117604   789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 24625050151f4b50aa8349c35184570a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.117877  1234 tablet_bootstrap.cc:654] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.123081   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "878b25ab3282423fab41014a8ea8949c" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:45.124661  2237 raft_consensus.cc:2241] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:45.125344  2237 raft_consensus.cc:2270] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:45.127353  2237 ts_tablet_manager.cc:1905] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:45.133252  2668 raft_consensus.cc:491] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.133783  2668 raft_consensus.cc:513] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.135659  1230 tablet_bootstrap.cc:492] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:45.136065  1230 ts_tablet_manager.cc:1397] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.511s	user 0.008s	sys 0.000s
I20250628 07:52:45.059870   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet fa3ec14241e344029df3e39473181078 (table default.Table_11e96390464141b9a308e3f956e1a84d [id=7a3e87e260074351a8dc381d5d01c58a]) successfully deleted
W20250628 07:52:45.142740   532 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 82887.9us
I20250628 07:52:45.144548   966 tablet_service.cc:1468] Processing CreateTablet for tablet 340b3ef1c06d48bc882e16e5765d4e22 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.145167  1107 tablet_service.cc:1468] Processing CreateTablet for tablet d96ec557ca9f4008846e8b99610174d5 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:45.145412   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 71c5bdface3d44ecae33bef018e7f18d (table default.Table_6fd35d7894eb4e9891f539a62d743e62 [id=700083322b4a4b118855057493210fb7], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 71c5bdface3d44ecae33bef018e7f18d consensus state has no leader
I20250628 07:52:45.146229  1107 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d96ec557ca9f4008846e8b99610174d5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.146332   543 catalog_manager.cc:5582] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:45.151383  1093 tablet_service.cc:1468] Processing CreateTablet for tablet 340b3ef1c06d48bc882e16e5765d4e22 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.153744  1093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340b3ef1c06d48bc882e16e5765d4e22. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.153514   800 tablet_service.cc:1468] Processing CreateTablet for tablet c6b42038b74a450796b43fa89d462017 (DEFAULT_TABLE table=default.Table_992ef1fbcdb840c382cc60a218166855 [id=868a11eb38d6484b98187acb9813872f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.154683   800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6b42038b74a450796b43fa89d462017. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.163525   803 tablet_service.cc:1468] Processing CreateTablet for tablet 9130fcc0c37f4d8e8852719fbafc3695 (DEFAULT_TABLE table=default.Table_fe7551bb42dd452ab32b051075e3398b [id=6ad3b42bbcbd4d5e894480654bf6bebe]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.164821   803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9130fcc0c37f4d8e8852719fbafc3695. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.164888  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f8bce65d32e042ab89ebe168d9657466" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:45.165915  1122 raft_consensus.cc:2391] T f8bce65d32e042ab89ebe168d9657466 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:45.167972  1103 tablet_service.cc:1468] Processing CreateTablet for tablet 928ffc3fd77d4ffe9c59c96afb436838 (DEFAULT_TABLE table=default.Table_f6dc476a6f3b42afa13683cc9611a247 [id=6514b4f905774cf3bfec0ddc73804534]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.169075  1103 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 928ffc3fd77d4ffe9c59c96afb436838. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:45.191955   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 131050us
I20250628 07:52:45.192833   966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340b3ef1c06d48bc882e16e5765d4e22. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.179329   793 tablet_service.cc:1468] Processing CreateTablet for tablet d200ea874eca4774a709c592a871e5ba (DEFAULT_TABLE table=default.Table_b6f2044e4f634f6182beb27ccdc55512 [id=11375d9db9b940f2bc7bfcb3c9904cac]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.180887   798 tablet_service.cc:1468] Processing CreateTablet for tablet 48b376ff0fbc47b086870e1503bd2d20 (DEFAULT_TABLE table=default.Table_0f6f46bde0ed463eb196dfdf0f049317 [id=6ceec3d9bfe142268afe00a134a83201]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.182353   801 tablet_service.cc:1468] Processing CreateTablet for tablet 71d772c683f94e79bd14bfc139357844 (DEFAULT_TABLE table=default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.199932  2694 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:45.200426  2694 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:45.182667  2683 raft_consensus.cc:491] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.183481   804 tablet_service.cc:1468] Processing CreateTablet for tablet 340b3ef1c06d48bc882e16e5765d4e22 (DEFAULT_TABLE table=default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.183969   805 tablet_service.cc:1468] Processing CreateTablet for tablet ad4a43b1e1074531ba41ed08827ffa38 (DEFAULT_TABLE table=default.Table_4107fb91f5704f38b337a443ee54704c [id=d4a373f636634fd3ac4b34d1f31ca380]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.184545   797 tablet_service.cc:1468] Processing CreateTablet for tablet d96ec557ca9f4008846e8b99610174d5 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.185734   795 tablet_service.cc:1468] Processing CreateTablet for tablet 928ffc3fd77d4ffe9c59c96afb436838 (DEFAULT_TABLE table=default.Table_f6dc476a6f3b42afa13683cc9611a247 [id=6514b4f905774cf3bfec0ddc73804534]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.202121  2683 raft_consensus.cc:513] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.202569  2694 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:45.206696   543 catalog_manager.cc:5582] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:45.208530  2683 leader_election.cc:290] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.210759   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 63115b7be4a94b639c18601d438ed947 (table default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]) successfully deleted
I20250628 07:52:45.211381   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:45.211429   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:45.213201  2059 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:45.218317  1062 leader_election.cc:304] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:45.219048  2658 raft_consensus.cc:2802] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:45.219024  2683 raft_consensus.cc:491] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.219362  2658 raft_consensus.cc:491] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:45.219683  2658 raft_consensus.cc:3058] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.219424  2683 raft_consensus.cc:513] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.219825  2694 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:45.221361  1229 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:45.221802  1229 ts_tablet_manager.cc:1397] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.567s	user 0.012s	sys 0.008s
I20250628 07:52:45.222800  2683 leader_election.cc:290] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.227017  1101 tablet_service.cc:1468] Processing CreateTablet for tablet b2bd3fccfe524ceaa7ff1a566ca72631 (DEFAULT_TABLE table=default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 [id=3dfaaca8b9384537917d5e8262b416d3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.227349  1105 tablet_service.cc:1468] Processing CreateTablet for tablet 5d5b34ea6d7740aca2fc9827f1fefc37 (DEFAULT_TABLE table=default.Table_af46d3e348f94f0fbd27c3f97a2028ff [id=93aca27bae214b44a27421372584676f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.228354   794 tablet_service.cc:1468] Processing CreateTablet for tablet b2bd3fccfe524ceaa7ff1a566ca72631 (DEFAULT_TABLE table=default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 [id=3dfaaca8b9384537917d5e8262b416d3]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:45.229691  1062 leader_election.cc:343] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:45.230219  1062 leader_election.cc:343] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:45.230701  1062 leader_election.cc:343] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.231019  1062 leader_election.cc:304] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.228951  2658 raft_consensus.cc:513] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.232638  2237 ts_tablet_manager.cc:1918] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:45.232976  2237 log.cc:1199] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/af13b56df157443c994f747e52b0f458
I20250628 07:52:45.233120  2658 leader_election.cc:290] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:45.233632  2237 ts_tablet_manager.cc:1939] T af13b56df157443c994f747e52b0f458 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:45.217065  1234 tablet_bootstrap.cc:492] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:45.234938  1234 ts_tablet_manager.cc:1397] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.126s	user 0.011s	sys 0.010s
I20250628 07:52:45.235777  1108 tablet_service.cc:1468] Processing CreateTablet for tablet ad4a43b1e1074531ba41ed08827ffa38 (DEFAULT_TABLE table=default.Table_4107fb91f5704f38b337a443ee54704c [id=d4a373f636634fd3ac4b34d1f31ca380]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.235842   796 tablet_service.cc:1468] Processing CreateTablet for tablet 5d5b34ea6d7740aca2fc9827f1fefc37 (DEFAULT_TABLE table=default.Table_af46d3e348f94f0fbd27c3f97a2028ff [id=93aca27bae214b44a27421372584676f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.244943   788 tablet_service.cc:1468] Processing CreateTablet for tablet 033bea0d32c048e29df6cc3443fc106a (DEFAULT_TABLE table=default.Table_14ad18ef5e224208bffdbc4a88018111 [id=4c9749c443364cee95d905b8c5973e3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.205302  2669 raft_consensus.cc:491] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.278223  2669 raft_consensus.cc:513] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.279925   961 tablet_service.cc:1468] Processing CreateTablet for tablet 5d5b34ea6d7740aca2fc9827f1fefc37 (DEFAULT_TABLE table=default.Table_af46d3e348f94f0fbd27c3f97a2028ff [id=93aca27bae214b44a27421372584676f]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.280771   957 tablet_service.cc:1468] Processing CreateTablet for tablet 928ffc3fd77d4ffe9c59c96afb436838 (DEFAULT_TABLE table=default.Table_f6dc476a6f3b42afa13683cc9611a247 [id=6514b4f905774cf3bfec0ddc73804534]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.280771   958 tablet_service.cc:1468] Processing CreateTablet for tablet ad4a43b1e1074531ba41ed08827ffa38 (DEFAULT_TABLE table=default.Table_4107fb91f5704f38b337a443ee54704c [id=d4a373f636634fd3ac4b34d1f31ca380]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.281788   957 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 928ffc3fd77d4ffe9c59c96afb436838. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.284025  1229 raft_consensus.cc:357] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.286578  1229 raft_consensus.cc:383] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.286962   961 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5d5b34ea6d7740aca2fc9827f1fefc37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.286917  1229 raft_consensus.cc:738] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.281765  2606 raft_consensus.cc:2747] T fc4c7770fb564ccd95b3c789b179eecf P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.289413   958 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4a43b1e1074531ba41ed08827ffa38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.290373   965 tablet_service.cc:1468] Processing CreateTablet for tablet b2bd3fccfe524ceaa7ff1a566ca72631 (DEFAULT_TABLE table=default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 [id=3dfaaca8b9384537917d5e8262b416d3]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.291290   965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bd3fccfe524ceaa7ff1a566ca72631. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.287686  1229 consensus_queue.cc:260] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.294864   793 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d200ea874eca4774a709c592a871e5ba. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:52:45.282454  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 51766.6us
W20250628 07:52:45.297459  1062 leader_election.cc:343] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.297819  1062 leader_election.cc:304] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.298463  2606 raft_consensus.cc:2747] T 878b25ab3282423fab41014a8ea8949c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.300138   758 leader_election.cc:304] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.300384   798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48b376ff0fbc47b086870e1503bd2d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.301445   805 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4a43b1e1074531ba41ed08827ffa38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.302111   801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 71d772c683f94e79bd14bfc139357844. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.301470  2688 raft_consensus.cc:2747] T f8bce65d32e042ab89ebe168d9657466 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.302690   797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d96ec557ca9f4008846e8b99610174d5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.303237   804 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340b3ef1c06d48bc882e16e5765d4e22. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.303931   795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 928ffc3fd77d4ffe9c59c96afb436838. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.308207   794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bd3fccfe524ceaa7ff1a566ca72631. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.285738  1230 raft_consensus.cc:357] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.337352  1230 raft_consensus.cc:383] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.337674  1230 raft_consensus.cc:738] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.338361   796 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5d5b34ea6d7740aca2fc9827f1fefc37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.338361  1230 consensus_queue.cc:260] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.339588  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 419) took 4032 ms. Trace:
I20250628 07:52:45.340243  1230 ts_tablet_manager.cc:1428] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.204s	user 0.006s	sys 0.001s
I20250628 07:52:45.339885  2237 rpcz_store.cc:276] 0628 07:52:41.306915 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.307050 (+   135us) service_pool.cc:225] Handling call
0628 07:52:45.111845 (+3804795us) ts_tablet_manager.cc:1145] Deleting tablet af13b56df157443c994f747e52b0f458
0628 07:52:45.111902 (+    57us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.191322 (+ 79420us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.232433 (+ 41111us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.339537 (+107104us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":23216,"tablet-delete.queue_time_us":3804270}
I20250628 07:52:45.340909  2237 tablet_replica.cc:331] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:45.341609   788 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 033bea0d32c048e29df6cc3443fc106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.346252   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:45.349620  2237 raft_consensus.cc:2241] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:45.350381  2237 raft_consensus.cc:2270] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:45.352890  2237 ts_tablet_manager.cc:1905] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:52:45.362251  1062 leader_election.cc:343] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.373584  2514 ts_tablet_manager.cc:1918] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:45.373960  2514 log.cc:1199] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/a447edf0960c4739930450a8c24ff6f7
I20250628 07:52:45.374622  2514 ts_tablet_manager.cc:1939] T a447edf0960c4739930450a8c24ff6f7 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:45.383790  1234 raft_consensus.cc:357] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.384388  1234 raft_consensus.cc:383] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.384714  1234 raft_consensus.cc:738] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.472812   978 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.473754   984 raft_consensus.cc:3058] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.473870  2668 leader_election.cc:290] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.474742  2669 leader_election.cc:290] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.475361   980 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.483026   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "181c57eeb50745beb09c36a38b75d9be" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:45.484377   984 raft_consensus.cc:3058] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.492110   959 tablet_service.cc:1468] Processing CreateTablet for tablet 033bea0d32c048e29df6cc3443fc106a (DEFAULT_TABLE table=default.Table_14ad18ef5e224208bffdbc4a88018111 [id=4c9749c443364cee95d905b8c5973e3e]), partition=RANGE (key) PARTITION UNBOUNDED
W20250628 07:52:45.507181   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 314168us
W20250628 07:52:45.507789   920 leader_election.cc:343] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.508620  2540 consensus_queue.cc:1035] T e52e15aefe194a61baa7e74616d9bfd3 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.279925   956 tablet_service.cc:1468] Processing CreateTablet for tablet d96ec557ca9f4008846e8b99610174d5 (DEFAULT_TABLE table=default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.513933  1234 consensus_queue.cc:260] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.531818  1234 ts_tablet_manager.cc:1428] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.296s	user 0.006s	sys 0.000s
I20250628 07:52:45.532693  1234 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:45.535245  1091 tablet_service.cc:1468] Processing CreateTablet for tablet 033bea0d32c048e29df6cc3443fc106a (DEFAULT_TABLE table=default.Table_14ad18ef5e224208bffdbc4a88018111 [id=4c9749c443364cee95d905b8c5973e3e]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:52:45.540036  2699 consensus_queue.cc:1035] T fb3d9cc430354e54b82ed7844fd2181c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.546886   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet af13b56df157443c994f747e52b0f458 (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]) successfully deleted
I20250628 07:52:45.547459  1122 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "fb3d9cc430354e54b82ed7844fd2181c" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:45.548461  1125 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e52e15aefe194a61baa7e74616d9bfd3" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:45.551517  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:45.552073  1129 raft_consensus.cc:2466] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:45.553633   758 leader_election.cc:304] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.554558  2683 raft_consensus.cc:2747] T fb3d9cc430354e54b82ed7844fd2181c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:45.554610   758 leader_election.cc:304] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.554697   921 leader_election.cc:304] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:45.555333  2540 raft_consensus.cc:2802] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:45.555642  2540 raft_consensus.cc:491] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:45.555953  2540 raft_consensus.cc:3058] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.555968  2686 raft_consensus.cc:2747] T e52e15aefe194a61baa7e74616d9bfd3 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:52:45.557192   984 raft_consensus.cc:2466] T 181c57eeb50745beb09c36a38b75d9be P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:45.562834  1091 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 033bea0d32c048e29df6cc3443fc106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.563860  1062 leader_election.cc:304] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:45.564816  2606 raft_consensus.cc:2802] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:45.565378  2606 raft_consensus.cc:695] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
I20250628 07:52:45.566190  2606 consensus_queue.cc:237] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.564968  1105 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5d5b34ea6d7740aca2fc9827f1fefc37. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.569311  1230 tablet_bootstrap.cc:492] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:45.517406  2237 ts_tablet_manager.cc:1918] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:45.522065  2688 consensus_queue.cc:1035] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.572887  2237 log.cc:1199] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c4fcefaa9e954b52bbbee6e0a2f2f939
I20250628 07:52:45.573621  2237 ts_tablet_manager.cc:1939] T c4fcefaa9e954b52bbbee6e0a2f2f939 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:45.573694  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 411) took 3049 ms. Trace:
I20250628 07:52:45.574021  2514 rpcz_store.cc:276] 0628 07:52:42.523984 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.524653 (+   669us) service_pool.cc:225] Handling call
0628 07:52:45.019699 (+2495046us) ts_tablet_manager.cc:1145] Deleting tablet a447edf0960c4739930450a8c24ff6f7
0628 07:52:45.019746 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.242599 (+222853us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.373361 (+130762us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.573643 (+200282us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":49277,"tablet-delete.queue_time_us":2494600}
I20250628 07:52:45.575340  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 412) took 2812 ms. Trace:
I20250628 07:52:45.575605  2514 rpcz_store.cc:276] 0628 07:52:42.763005 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.763143 (+   138us) service_pool.cc:225] Handling call
0628 07:52:45.575046 (+2811903us) ts_tablet_manager.cc:1145] Deleting tablet 88d4a84c8ed7475c9bf043aaba8ad624
0628 07:52:45.575081 (+    35us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.575293 (+   212us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2811296}
I20250628 07:52:45.576560  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 413) took 2726 ms. Trace:
W20250628 07:52:45.557335   921 leader_election.cc:343] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: a8518c82c188484c8554d3115637f5be
I20250628 07:52:45.578730  1101 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bd3fccfe524ceaa7ff1a566ca72631. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.578640   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet a447edf0960c4739930450a8c24ff6f7 (table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]) successfully deleted
W20250628 07:52:45.579273   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 88d4a84c8ed7475c9bf043aaba8ad624 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 88d4a84c8ed7475c9bf043aaba8ad624 already in progress: creating tablet
W20250628 07:52:45.580495   758 consensus_peers.cc:489] T a8518c82c188484c8554d3115637f5be P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a8518c82c188484c8554d3115637f5be. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:45.581957  1230 tablet_bootstrap.cc:654] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:45.587297   758 consensus_peers.cc:489] T a447edf0960c4739930450a8c24ff6f7 P 6005c44dfe4848f18a304621cd49e934 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a447edf0960c4739930450a8c24ff6f7. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:45.576838  2514 rpcz_store.cc:276] 0628 07:52:42.850365 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.850528 (+   163us) service_pool.cc:225] Handling call
0628 07:52:45.576339 (+2725811us) ts_tablet_manager.cc:1145] Deleting tablet c4fcefaa9e954b52bbbee6e0a2f2f939
0628 07:52:45.576372 (+    33us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.576517 (+   145us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2725215}
I20250628 07:52:45.588771  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 415) took 1987 ms. Trace:
W20250628 07:52:45.590306   533 catalog_manager.cc:4895] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet c4fcefaa9e954b52bbbee6e0a2f2f939 because the tablet was not found. No further retry: Not found: Tablet not found: c4fcefaa9e954b52bbbee6e0a2f2f939
I20250628 07:52:45.591066  1108 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4a43b1e1074531ba41ed08827ffa38. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.591637  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 420) took 4276 ms. Trace:
I20250628 07:52:45.591930  2237 rpcz_store.cc:276] 0628 07:52:41.315003 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.315125 (+   122us) service_pool.cc:225] Handling call
0628 07:52:45.340759 (+4025634us) ts_tablet_manager.cc:1145] Deleting tablet c4fcefaa9e954b52bbbee6e0a2f2f939
0628 07:52:45.340812 (+    53us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.380476 (+ 39664us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.517184 (+136708us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.591582 (+ 74398us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4025147}
I20250628 07:52:45.589059  2514 rpcz_store.cc:276] 0628 07:52:43.601065 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.601623 (+   558us) service_pool.cc:225] Handling call
0628 07:52:45.588437 (+1986814us) ts_tablet_manager.cc:1145] Deleting tablet 51a44ec161684f74b228830ff06f1957
0628 07:52:45.588477 (+    40us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.588716 (+   239us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1986205}
I20250628 07:52:45.593807  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 423) took 1593 ms. Trace:
I20250628 07:52:45.594134  2514 rpcz_store.cc:276] 0628 07:52:44.000121 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:44.002125 (+  2004us) service_pool.cc:225] Handling call
0628 07:52:45.593485 (+1591360us) ts_tablet_manager.cc:1145] Deleting tablet 670f91ee94f7460caa0790e232ac7fa8
0628 07:52:45.593529 (+    44us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.593751 (+   222us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1577101}
I20250628 07:52:45.595438  2514 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54948 (request call id 422) took 1598 ms. Trace:
I20250628 07:52:45.595897  2514 rpcz_store.cc:276] 0628 07:52:43.997298 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:44.000298 (+  3000us) service_pool.cc:225] Handling call
0628 07:52:45.595160 (+1594862us) ts_tablet_manager.cc:1145] Deleting tablet 6ce66199d1614826bf618819f6928383
0628 07:52:45.595197 (+    37us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.595394 (+   197us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1577606}
W20250628 07:52:45.596737   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 51a44ec161684f74b228830ff06f1957 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 51a44ec161684f74b228830ff06f1957 already in progress: creating tablet
W20250628 07:52:45.597393   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 670f91ee94f7460caa0790e232ac7fa8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 670f91ee94f7460caa0790e232ac7fa8 already in progress: creating tablet
W20250628 07:52:45.598470   533 catalog_manager.cc:4908] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): delete failed for tablet 6ce66199d1614826bf618819f6928383 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 6ce66199d1614826bf618819f6928383 already in progress: creating tablet
I20250628 07:52:45.604108   959 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 033bea0d32c048e29df6cc3443fc106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.609326   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:45.612892   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a8518c82c188484c8554d3115637f5be" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:45.615602  1234 tablet_bootstrap.cc:654] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.617417  1230 tablet_bootstrap.cc:492] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:45.617841  1230 ts_tablet_manager.cc:1397] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.049s	user 0.007s	sys 0.002s
I20250628 07:52:45.620028  1230 raft_consensus.cc:357] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.620713  1230 raft_consensus.cc:383] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.620988  1230 raft_consensus.cc:738] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.621666  1230 consensus_queue.cc:260] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:45.622145   921 consensus_peers.cc:489] T 29a77cd9dc244d239a518e004c49e3c8 P 82f43cdfab544b17a6a71033983e65d4 -> Peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Couldn't send request to peer 6a58514e01ea4989809bc2992fb2b806. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 29a77cd9dc244d239a518e004c49e3c8. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:52:45.623823   956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d96ec557ca9f4008846e8b99610174d5. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:52:45.623983  1230 ts_tablet_manager.cc:1428] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20250628 07:52:45.624812  1230 tablet_bootstrap.cc:492] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:45.625358  1229 ts_tablet_manager.cc:1428] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.403s	user 0.004s	sys 0.010s
I20250628 07:52:45.626190  1229 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:45.631217  2540 raft_consensus.cc:513] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.632011  2657 consensus_queue.cc:1035] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.641511   818 raft_consensus.cc:3058] T 5f77a64b8e054f45acbad44d02af0e82 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.646657  2606 raft_consensus.cc:491] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.647119  2606 raft_consensus.cc:513] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.651747  2606 leader_election.cc:290] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:45.654954   825 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e4bf0710bcf145a382516afd4ee09f0b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:45.656036  1062 leader_election.cc:343] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.665715  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 422) took 4244 ms. Trace:
I20250628 07:52:45.666033  2237 rpcz_store.cc:276] 0628 07:52:41.420910 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.421071 (+   161us) service_pool.cc:225] Handling call
0628 07:52:45.665301 (+4244230us) ts_tablet_manager.cc:1145] Deleting tablet 06b9b63649384313948964b3c0dd11cf
0628 07:52:45.665352 (+    51us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.665661 (+   309us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4242984}
I20250628 07:52:45.667119  2237 tablet_replica.cc:331] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:45.676215  2606 raft_consensus.cc:491] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.676769  2606 raft_consensus.cc:513] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.680022  2606 leader_election.cc:290] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:45.601061   544 catalog_manager.cc:5674] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:45.670840   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet c4fcefaa9e954b52bbbee6e0a2f2f939 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]) successfully deleted
W20250628 07:52:45.694727   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 06b9b63649384313948964b3c0dd11cf because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 06b9b63649384313948964b3c0dd11cf already in progress: creating tablet
W20250628 07:52:45.697187   920 leader_election.cc:343] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.697446   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "19e7760b95b94336a985cdd128cef8de" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:45.698582  1062 leader_election.cc:343] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:45.528894   954 consensus_meta.cc:287] T 12d856ea75ef4f558efc58afd73339f4 P 82f43cdfab544b17a6a71033983e65d4: Time spent flushing consensus metadata: real 0.684s	user 0.012s	sys 0.014s
I20250628 07:52:45.707439   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e4bf0710bcf145a382516afd4ee09f0b" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.725039  2237 raft_consensus.cc:2241] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:45.725896  2237 raft_consensus.cc:2270] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:45.727996  2237 ts_tablet_manager.cc:1905] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:45.741037  1230 tablet_bootstrap.cc:654] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.744302  1229 tablet_bootstrap.cc:654] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.749529  1234 tablet_bootstrap.cc:492] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:45.750025  1234 ts_tablet_manager.cc:1397] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.217s	user 0.009s	sys 0.006s
I20250628 07:52:45.752357  1234 raft_consensus.cc:357] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.762442  1230 tablet_bootstrap.cc:492] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:45.762838  1230 ts_tablet_manager.cc:1397] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.138s	user 0.008s	sys 0.000s
I20250628 07:52:45.764931  1230 raft_consensus.cc:357] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.765529  1230 raft_consensus.cc:383] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.765755  1230 raft_consensus.cc:738] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.767606  1234 raft_consensus.cc:383] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.767874  1234 raft_consensus.cc:738] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
W20250628 07:52:45.784786  1062 leader_election.cc:343] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.785188  1062 leader_election.cc:304] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.786108  2658 raft_consensus.cc:2747] T e4bf0710bcf145a382516afd4ee09f0b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.789330  1230 consensus_queue.cc:260] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.790050  2606 raft_consensus.cc:491] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.790438  2606 raft_consensus.cc:513] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.790827  1234 consensus_queue.cc:260] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.792724  2606 leader_election.cc:290] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:45.758901   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 60715.8us
I20250628 07:52:45.798350   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "81d9066872654fceb44b7fb63b60f070" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:45.799870  1062 leader_election.cc:343] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.802944   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "19e7760b95b94336a985cdd128cef8de" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.803692  2714 raft_consensus.cc:491] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.804118  2714 raft_consensus.cc:513] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.810928  1230 ts_tablet_manager.cc:1428] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.048s	user 0.006s	sys 0.000s
W20250628 07:52:45.813138  1062 leader_election.cc:343] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.813503  1062 leader_election.cc:304] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.814355  2606 raft_consensus.cc:2747] T 19e7760b95b94336a985cdd128cef8de P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.819267  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:45.819784  1129 raft_consensus.cc:2466] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:45.822824   921 leader_election.cc:304] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:45.836092  2699 raft_consensus.cc:2802] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:45.836493  2699 raft_consensus.cc:491] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:45.836876  2699 raft_consensus.cc:3058] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.842253  1234 ts_tablet_manager.cc:1428] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.092s	user 0.006s	sys 0.000s
I20250628 07:52:45.836068  2714 leader_election.cc:290] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.851279  1230 tablet_bootstrap.cc:492] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:45.854861  1234 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:45.856618  1230 tablet_bootstrap.cc:654] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.861371   979 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "81d9066872654fceb44b7fb63b60f070" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:45.861330  2701 consensus_queue.cc:1035] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:45.874112  1234 tablet_bootstrap.cc:654] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:45.877436  2717 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:45.879065  2717 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:45.882027  2540 leader_election.cc:290] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.886200  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:45.886783  1124 raft_consensus.cc:3058] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.886863  2721 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:45.887253  2721 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:45.891111  1230 tablet_bootstrap.cc:492] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:45.891492  1230 ts_tablet_manager.cc:1397] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.040s	user 0.007s	sys 0.002s
I20250628 07:52:45.895182  1124 raft_consensus.cc:2466] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:45.898938  2540 consensus_queue.cc:1035] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.912029  2142 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:45.912628  2142 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 3.
I20250628 07:52:45.913980   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276, 992ac4c27fb744deb1edf7765553ef08; no voters: 
I20250628 07:52:45.917910  2699 raft_consensus.cc:513] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.935535  2717 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:45.921051  1117 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:45.940600  1117 raft_consensus.cc:3058] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:45.940827   921 leader_election.cc:304] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
W20250628 07:52:45.937338  1062 leader_election.cc:343] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:45.941625  2702 raft_consensus.cc:2802] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:45.941864  1062 leader_election.cc:304] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:45.942083  2702 raft_consensus.cc:695] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:45.942759  2702 consensus_queue.cc:237] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:45.941614   920 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:34311, user_credentials={real_user=slave}} blocked reactor thread for 143529us
I20250628 07:52:45.948130   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:45.941500  2717 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election won for term 4
I20250628 07:52:45.950301   636 raft_consensus.cc:2408] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5fd67221fa5f424ea0f83e582629d276 for term 4 because replica has last-logged OpId of term: 3 index: 555, which is greater than that of the candidate, which has last-logged OpId of term: 3 index: 417.
I20250628 07:52:45.954332   920 leader_election.cc:304] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934, 6a58514e01ea4989809bc2992fb2b806
I20250628 07:52:45.955071  2717 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 023ed4cb582a44509bccf42b52c494d7)
I20250628 07:52:45.956427  2717 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:52:45.958840  2702 consensus_queue.cc:1035] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:45.963675  1229 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:45.964027  1229 ts_tablet_manager.cc:1397] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.338s	user 0.005s	sys 0.007s
W20250628 07:52:45.964821   610 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.190:37209, user_credentials={real_user=slave}} blocked reactor thread for 50996.9us
I20250628 07:52:45.965183  2699 leader_election.cc:290] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.968874   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:45.970233   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5fd67221fa5f424ea0f83e582629d276" candidate_term: 4 candidate_status { last_received { term: 3 index: 417 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:45.972062  2059 leader_election.cc:304] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5fd67221fa5f424ea0f83e582629d276; no voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08
I20250628 07:52:45.983620  1117 raft_consensus.cc:2466] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:45.986377  1234 tablet_bootstrap.cc:492] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:45.986770  1234 ts_tablet_manager.cc:1397] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.132s	user 0.013s	sys 0.009s
I20250628 07:52:45.987777  2718 raft_consensus.cc:2747] T a8518c82c188484c8554d3115637f5be P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:45.988700  2660 raft_consensus.cc:491] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:45.988910  1234 raft_consensus.cc:357] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.989600  1234 raft_consensus.cc:383] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:45.989884  1234 raft_consensus.cc:738] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:45.989609  2660 raft_consensus.cc:513] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.990253   921 leader_election.cc:304] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:45.990437  1234 consensus_queue.cc:260] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.991487  2660 leader_election.cc:290] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:45.991991  2714 raft_consensus.cc:2802] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:45.992406  1234 ts_tablet_manager.cc:1428] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
I20250628 07:52:45.992352  2714 raft_consensus.cc:695] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:45.993170  2714 consensus_queue.cc:237] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:45.888993  2721 leader_election.cc:290] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:45.995898  2721 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:52:45.999145   824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:45.999591   825 raft_consensus.cc:3058] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.003999   817 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "613eb4aafd7b4a93b0f2a9699a942f34" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:46.009274   818 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:46.019292  1234 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.020215   983 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:46.022529  1229 raft_consensus.cc:357] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.023355  1229 raft_consensus.cc:383] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.023620  1229 raft_consensus.cc:738] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.024243  1229 consensus_queue.cc:260] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.026808  1229 ts_tablet_manager.cc:1428] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.063s	user 0.006s	sys 0.001s
I20250628 07:52:46.027791  1229 tablet_bootstrap.cc:492] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:46.031396  1234 tablet_bootstrap.cc:654] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.031198  2717 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:46.032410   636 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 FOLLOWER]: Rejecting Update request from peer 023ed4cb582a44509bccf42b52c494d7 for earlier term 3. Current term is 4. Ops: []
I20250628 07:52:46.038344  2722 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 }, Status: INVALID_TERM, Last received: 3.555, Next index: 556, Last known committed idx: 555, Time since last communication: 0.001s
I20250628 07:52:46.047905  1230 raft_consensus.cc:357] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:46.048460  1062 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.130:35061, user_credentials={real_user=slave}} blocked reactor thread for 111141us
I20250628 07:52:46.048629  1230 raft_consensus.cc:383] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.050837  1124 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "3519e872e8504a4fb962814e3438bc01" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.051664  2658 raft_consensus.cc:2747] T 81d9066872654fceb44b7fb63b60f070 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:46.054661   758 leader_election.cc:304] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:46.057298  2737 raft_consensus.cc:3053] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 LEADER]: Stepping down as leader of term 3
I20250628 07:52:46.058315  2440 raft_consensus.cc:2747] T 3519e872e8504a4fb962814e3438bc01 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:46.062309  2737 raft_consensus.cc:738] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 LEADER]: Becoming Follower/Learner. State: Replica: 023ed4cb582a44509bccf42b52c494d7, State: Running, Role: LEADER
W20250628 07:52:46.036094   920 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.063367  2737 consensus_queue.cc:260] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 555, Committed index: 555, Last appended: 3.555, Last appended by leader: 555, Current term: 3, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:46.065369  2737 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:52:45.974139  2237 ts_tablet_manager.cc:1918] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20250628 07:52:46.068648  2237 log.cc:1199] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/22b47b39f8eb45a0b5d2c5348bfbc55c
I20250628 07:52:46.071480   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]). Expected version 2 got 0
I20250628 07:52:46.081830  2237 ts_tablet_manager.cc:1939] T 22b47b39f8eb45a0b5d2c5348bfbc55c P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:46.085729  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 424) took 4281 ms. Trace:
I20250628 07:52:46.086112  2237 rpcz_store.cc:276] 0628 07:52:41.803884 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.804025 (+   141us) service_pool.cc:225] Handling call
0628 07:52:45.666942 (+3862917us) ts_tablet_manager.cc:1145] Deleting tablet 22b47b39f8eb45a0b5d2c5348bfbc55c
0628 07:52:45.667002 (+    60us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:45.922484 (+255482us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:45.973850 (+ 51366us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:46.085678 (+111828us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":56917,"spinlock_wait_cycles":22912,"tablet-delete.queue_time_us":3862334}
I20250628 07:52:46.087213  2714 raft_consensus.cc:491] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.087611  2714 raft_consensus.cc:513] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.090972  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.091483  1127 raft_consensus.cc:2466] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:46.092422   921 leader_election.cc:304] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:46.093322  2540 raft_consensus.cc:2802] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:46.093727  2540 raft_consensus.cc:491] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:46.094116  2540 raft_consensus.cc:3058] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.048859  1230 raft_consensus.cc:738] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.095561  1230 consensus_queue.cc:260] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.098325  1230 ts_tablet_manager.cc:1428] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.207s	user 0.007s	sys 0.000s
I20250628 07:52:46.102064  2540 raft_consensus.cc:513] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.105190  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:46.105003  2540 leader_election.cc:290] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.105659  1127 raft_consensus.cc:3058] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:46.107296   920 leader_election.cc:343] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.111544  1127 raft_consensus.cc:2466] T d9e0ea5e300347c7adc8e30b9070976b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:46.112516   921 leader_election.cc:304] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:46.117038  2715 raft_consensus.cc:2802] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:46.117524  2715 raft_consensus.cc:695] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:46.118326  2715 consensus_queue.cc:237] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.128512  2717 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:46.129395  2142 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:46.129940  2142 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:52:46.133410   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 4 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7"
I20250628 07:52:46.140120   555 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 4.
I20250628 07:52:46.141419   613 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 992ac4c27fb744deb1edf7765553ef08; no voters: 
I20250628 07:52:46.145327  2717 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 FOLLOWER]: Leader election won for term 4
I20250628 07:52:46.146008  2717 raft_consensus.cc:695] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 LEADER]: Becoming Leader. State: Replica: 992ac4c27fb744deb1edf7765553ef08, State: Running, Role: LEADER
I20250628 07:52:46.146847  2717 consensus_queue.cc:237] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 555, Committed index: 555, Last appended: 3.555, Last appended by leader: 555, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:46.151839  2731 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 992ac4c27fb744deb1edf7765553ef08. Latest consensus state: current_term: 4 leader_uuid: "992ac4c27fb744deb1edf7765553ef08" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:46.152522  2731 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:46.156466  2142 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 992ac4c27fb744deb1edf7765553ef08 in term 4.
W20250628 07:52:46.159102   920 leader_election.cc:343] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.167150  1234 tablet_bootstrap.cc:492] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.167598  1234 ts_tablet_manager.cc:1397] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.175s	user 0.022s	sys 0.008s
I20250628 07:52:46.169663  2237 tablet_replica.cc:331] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:46.170491  2237 raft_consensus.cc:2241] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:46.170835  1234 raft_consensus.cc:357] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.166276  2740 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:52:46.171896   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 22b47b39f8eb45a0b5d2c5348bfbc55c (table default.Table_df287bcea5bd414ebe5aee1689733f8c [id=398556bc3bdc481cb6577321cbc31055]) successfully deleted
I20250628 07:52:46.171479  1234 raft_consensus.cc:383] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.172578  1230 tablet_bootstrap.cc:492] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:46.172681  1234 raft_consensus.cc:738] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.173424  1234 consensus_queue.cc:260] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.171514  2237 raft_consensus.cc:2270] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:46.179872  2237 ts_tablet_manager.cc:1905] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:46.164188  2701 consensus_queue.cc:1035] T 3519e872e8504a4fb962814e3438bc01 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.019s
I20250628 07:52:46.186005   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:46.181434  1234 ts_tablet_manager.cc:1428] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20250628 07:52:46.136518  2714 leader_election.cc:290] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.191867   821 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
I20250628 07:52:46.192430   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d9e0ea5e300347c7adc8e30b9070976b" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:46.195866   920 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:46.196606   920 leader_election.cc:343] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.220088  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3b7ef98c85564dc9b40201e1713b7c55 [id=010e62c6a9a649d78b828d7a7dd2296f]
W20250628 07:52:46.221403   531 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 2595308cbc0d44f0abfd0a126f16d4df (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 2595308cbc0d44f0abfd0a126f16d4df consensus state has no leader
I20250628 07:52:46.223932  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_481f2aef57d748fa98e48e26b106b1f5 [id=0335cd54e0504a65ad54d973365cd75b]
I20250628 07:52:46.225332  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_5ad5f13abefc445b8c69ab0be547644d [id=033b1fae1f634663a3efa53e00c858ac]
I20250628 07:52:46.227309  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c81c2135bc5a4417b706d1c042207cc0 [id=0397ad0e0e114f359480497eac5e7712]
I20250628 07:52:46.229566  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b]
I20250628 07:52:46.231060  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_df7fc4695669432f9b2bcc300360c70b [id=065317d32ef7414aa62609e873462803]
I20250628 07:52:46.212929  2699 consensus_queue.cc:1035] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.020s
I20250628 07:52:46.233055  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_affcc9f556b943238389e6f1e205745b [id=070d78e5d6f54ab9a72d680f38247ac1]
I20250628 07:52:46.238138  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_324d60f6b2914fdb85b95e4201ffee43 [id=0b6ea0c021b3450ba01083f4373e29e8]
I20250628 07:52:46.239398  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_bc0991297d684c8fb6295c5ed29cd3c1 [id=0bcb503dea9b4b9b96ae9a3f6bb9f9a5]
I20250628 07:52:46.240713  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]
I20250628 07:52:46.242048  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e]
I20250628 07:52:46.243371  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_b6f2044e4f634f6182beb27ccdc55512 [id=11375d9db9b940f2bc7bfcb3c9904cac]
I20250628 07:52:46.244706  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_9bd0a6cbc9e8466cb54f871dc9938e95 [id=1234c3ae728a4b9fb829fed377c84833]
I20250628 07:52:46.246093  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_2780965e7e5d4d528b1d63995873222f [id=127fda336df94c48a181d1a59e0d2f00]
I20250628 07:52:46.247406  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d3fc569c0e854f788cd8355607639d3a [id=12c2d9bf9bc8422a9f415382c8202daa]
I20250628 07:52:46.247897  1229 tablet_bootstrap.cc:654] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.250221  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e04e4751022f4e2fbc686f49b598150e [id=14de477c143d4d2084393ef5d658e4d0]
I20250628 07:52:46.257200   547 catalog_manager.cc:5674] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709) does not have the latest schema for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]). Expected version 1 got 0
I20250628 07:52:46.259339  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1]
I20250628 07:52:46.260718  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_9775925de3b34654b11d221bc66cb7fd [id=1f4b61bbe4284a718e2adc4e1a1fcc59]
I20250628 07:52:46.262866  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_168a40d10a3645bdaea410044e99f357 [id=2013562180c34579a816dde8278e9c85]
I20250628 07:52:46.264216  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf [id=222109d2472a4f079bc6e4abf9423ee6]
I20250628 07:52:46.275053  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 [id=2609d3d6796f4fa0b9cb750fa925da21]
I20250628 07:52:46.276461  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_1cf78a5cb3c5490bae2317600c643635 [id=2613a92dda0d4236b4a4422cf612a6c6]
I20250628 07:52:46.281309  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7ac89842d7e54a2ea03d7f64f4a92175 [id=28124975e8e246c68bd570b13fa5af2a]
I20250628 07:52:46.286015  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15]
I20250628 07:52:46.287983  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_bda937abf44b485aa9fd46ab195442f8 [id=2a211e6249e24fc89a60604f187a5554]
I20250628 07:52:46.290432  2748 tablet.cc:1722] T 5cacde13bc29458da81f29fb4fa2945a P 82f43cdfab544b17a6a71033983e65d4: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:46.293587  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_ade87cf74e6b498690118e3541e41383 [id=2ccbb0163b564f39971348853bdbd012]
I20250628 07:52:46.294903  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c22cd1bf6d884c33bf39513fb5da717a [id=2f1a522e39a24e749aaaa0578ad3c6c2]
I20250628 07:52:46.297451  2749 tablet.cc:1722] T 5cacde13bc29458da81f29fb4fa2945a P 6005c44dfe4848f18a304621cd49e934: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:46.305836  2237 ts_tablet_manager.cc:1918] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:46.306201  2237 log.cc:1199] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/8155990eebd84fcf9ebc357e186d1bf2
I20250628 07:52:46.306890  2237 ts_tablet_manager.cc:1939] T 8155990eebd84fcf9ebc357e186d1bf2 P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:46.307539  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_cc2df6969d8d43ba882f1832a7c4b528 [id=31b9ad86edb24981b8fbe350f2878d9a]
I20250628 07:52:46.308784  1230 tablet_bootstrap.cc:654] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.308928  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e393a3a8856046f1a8d11d7ab8c485ee [id=3311e522b275451fa31744d77fcc8654]
I20250628 07:52:46.310294  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e13a07ffb7754b44a1e224a0cc1e4289 [id=3594565eae314c6e861c681fb61092bf]
I20250628 07:52:46.318789  1234 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.313081  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_b66cef7061354731bda56976a11f758d [id=380a32248ae74b7697746c5853248303]
I20250628 07:52:46.323900  1234 tablet_bootstrap.cc:654] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.324066  2606 consensus_queue.cc:1035] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:46.325300  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a]
I20250628 07:52:46.330114  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_6541827cef9a442685f4b25c2539eb33 [id=38e154d3c3fc455eb314ff2d8323bb5b]
I20250628 07:52:46.332082  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 426) took 4502 ms. Trace:
I20250628 07:52:46.332182  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_a5aaafa5f2c84037b58a9040e19581dd [id=390ca1dfaf6a44b193876507a9dbe929]
I20250628 07:52:46.332405  2237 rpcz_store.cc:276] 0628 07:52:41.829618 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.830053 (+   435us) service_pool.cc:225] Handling call
0628 07:52:46.169502 (+4339449us) ts_tablet_manager.cc:1145] Deleting tablet 8155990eebd84fcf9ebc357e186d1bf2
0628 07:52:46.169558 (+    56us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.249314 (+ 79756us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:46.305630 (+ 56316us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:46.332028 (+ 26398us) inbound_call.cc:173] Queueing success response
Metrics: {"mutex_wait_us":386,"tablet-delete.queue_time_us":4338610}
I20250628 07:52:46.333647   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet 8155990eebd84fcf9ebc357e186d1bf2 (table default.Table_39092c471b2443d183b7229a1d625b66 [id=ab01e12b48e047828205036a96ce6762]) successfully deleted
I20250628 07:52:46.334883  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 428) took 4501 ms. Trace:
I20250628 07:52:46.335268  2237 rpcz_store.cc:276] 0628 07:52:41.833335 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.833455 (+   120us) service_pool.cc:225] Handling call
0628 07:52:46.334502 (+4501047us) ts_tablet_manager.cc:1145] Deleting tablet e15d063a4bca40d480cba08733ec1ec3
0628 07:52:46.334570 (+    68us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.334804 (+   234us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4500505}
W20250628 07:52:46.336480   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet e15d063a4bca40d480cba08733ec1ec3 because the tablet was not found. No further retry: Not found: Tablet not found: e15d063a4bca40d480cba08733ec1ec3
I20250628 07:52:46.341632  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 429) took 4506 ms. Trace:
I20250628 07:52:46.342078  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83]
I20250628 07:52:46.341917  2237 rpcz_store.cc:276] 0628 07:52:41.834838 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.834992 (+   154us) service_pool.cc:225] Handling call
0628 07:52:46.341290 (+4506298us) ts_tablet_manager.cc:1145] Deleting tablet 5fa4299a9e0f4b409d04c8c5900901c0
0628 07:52:46.341348 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.341586 (+   238us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4505595}
W20250628 07:52:46.342870   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 5fa4299a9e0f4b409d04c8c5900901c0 because the tablet was not found. No further retry: Not found: Tablet not found: 5fa4299a9e0f4b409d04c8c5900901c0
I20250628 07:52:46.348580  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 427) took 4515 ms. Trace:
I20250628 07:52:46.348857  2237 rpcz_store.cc:276] 0628 07:52:41.832800 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:41.836280 (+  3480us) service_pool.cc:225] Handling call
0628 07:52:46.348326 (+4512046us) ts_tablet_manager.cc:1145] Deleting tablet 8e3e0e52b4924e67904d3acfa4dd5752
0628 07:52:46.348377 (+    51us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.348535 (+   158us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4511361}
I20250628 07:52:46.350678  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 [id=3dfaaca8b9384537917d5e8262b416d3]
W20250628 07:52:46.351076   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 8e3e0e52b4924e67904d3acfa4dd5752 because the tablet was not found. No further retry: Not found: Tablet not found: 8e3e0e52b4924e67904d3acfa4dd5752
I20250628 07:52:46.352032  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_577ffb94469047e2a14ccb8d89c9a4d6 [id=3e63dedd51ba4d36bd5f7d53a2dd94e4]
I20250628 07:52:46.353964  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_177b8783c82149aabfddcfe195d451a7 [id=3f993fcff5574851b92244167bb2ee8e]
I20250628 07:52:46.355243  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c35cbc8840ec40728f62ffc4fb20ce69 [id=41902d471a304c5d985c48efbe81acd0]
I20250628 07:52:46.356580  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7437789ae57b4f029d85e0722aa8fa84 [id=442671f219f54e26b6f9d8958301fbd8]
I20250628 07:52:46.360527  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_948ee6f5274e4638b253dcf9f5df2e31 [id=4454ad1ffc2744a1b55d3772f0722258]
I20250628 07:52:46.364737  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89]
I20250628 07:52:46.366012  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7631c6a0720642949d4b63e271a92e4c [id=46657c237cc041279dbb9e365d574c01]
I20250628 07:52:46.369079  1234 tablet_bootstrap.cc:492] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.369158  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e]
I20250628 07:52:46.369509  1234 ts_tablet_manager.cc:1397] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.051s	user 0.011s	sys 0.000s
I20250628 07:52:46.370409   821 raft_consensus.cc:3058] T 181c57eeb50745beb09c36a38b75d9be P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.371673  1234 raft_consensus.cc:357] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.372270  1234 raft_consensus.cc:383] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.372526  1234 raft_consensus.cc:738] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.357692  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 437) took 4147 ms. Trace:
I20250628 07:52:46.373710  2237 rpcz_store.cc:276] 0628 07:52:42.210673 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.210816 (+   143us) service_pool.cc:225] Handling call
0628 07:52:46.357307 (+4146491us) ts_tablet_manager.cc:1145] Deleting tablet fd67d7d9ba9446c19f1711fb5bd73ebd
0628 07:52:46.357368 (+    61us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.357646 (+   278us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4119142}
W20250628 07:52:46.374848   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet fd67d7d9ba9446c19f1711fb5bd73ebd because tablet deleting was already in progress. No further retry: Already present: State transition of tablet fd67d7d9ba9446c19f1711fb5bd73ebd already in progress: creating tablet
I20250628 07:52:46.375175  1229 tablet_bootstrap.cc:492] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:46.375545  1229 ts_tablet_manager.cc:1397] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.348s	user 0.004s	sys 0.005s
I20250628 07:52:46.375530  1234 consensus_queue.cc:260] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.376865  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 438) took 4125 ms. Trace:
I20250628 07:52:46.378851  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_998846661d2441bfba53057917d7f1bf [id=48ef616b338f4d7b932257bfb1b9f0be]
I20250628 07:52:46.378865  2658 consensus_queue.cc:1035] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:46.382588  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_bb80571048c94b099e096f7c712af383 [id=4a58d7983dd146ae9e4ae45df899b0b6]
I20250628 07:52:46.383159  1234 ts_tablet_manager.cc:1428] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20250628 07:52:46.384280  1234 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.385210  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_6c95fbef63ae4a34aff4518c2d8cbdad [id=4accce061ee5447187ab61870ffc1e77]
I20250628 07:52:46.377185  2237 rpcz_store.cc:276] 0628 07:52:42.251861 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.251968 (+   107us) service_pool.cc:225] Handling call
0628 07:52:46.376503 (+4124535us) ts_tablet_manager.cc:1145] Deleting tablet c3d1bc8483374a2a917451a43b3d42ed
0628 07:52:46.376561 (+    58us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.376816 (+   255us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4123890}
W20250628 07:52:46.386672   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet c3d1bc8483374a2a917451a43b3d42ed because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c3d1bc8483374a2a917451a43b3d42ed already in progress: creating tablet
I20250628 07:52:46.387624  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 443) took 3899 ms. Trace:
I20250628 07:52:46.387835  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_14ad18ef5e224208bffdbc4a88018111 [id=4c9749c443364cee95d905b8c5973e3e]
I20250628 07:52:46.387938  2237 rpcz_store.cc:276] 0628 07:52:42.488685 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.489158 (+   473us) service_pool.cc:225] Handling call
0628 07:52:46.387342 (+3898184us) ts_tablet_manager.cc:1145] Deleting tablet 9ba2a10acdf0462bb05628351cdbbbe4
0628 07:52:46.387394 (+    52us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.387575 (+   181us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3897319}
W20250628 07:52:46.389056   532 catalog_manager.cc:4895] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 9ba2a10acdf0462bb05628351cdbbbe4 because the tablet was not found. No further retry: Not found: Tablet not found: 9ba2a10acdf0462bb05628351cdbbbe4
I20250628 07:52:46.389915  2237 tablet_replica.cc:331] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
I20250628 07:52:46.389611  2754 tablet.cc:1722] T 5cacde13bc29458da81f29fb4fa2945a P 6a58514e01ea4989809bc2992fb2b806: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20250628 07:52:46.390317  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a]
I20250628 07:52:46.390648  2237 raft_consensus.cc:2241] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:52:46.392513  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_0b4dcd93c24b4479ad87780460b6afd8 [id=4fe25f3f36eb4a76addfc9f54bb85371]
I20250628 07:52:46.393723  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288]
I20250628 07:52:46.394348  1229 raft_consensus.cc:357] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.395061  1229 raft_consensus.cc:383] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.395320  1229 raft_consensus.cc:738] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.396230  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013]
I20250628 07:52:46.397405  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_f5b1b57d6e754cccb6eb0a1405289fde [id=53bd74ded2bf4f5fa0edb44f63d4a340]
I20250628 07:52:46.398495  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 [id=54d01002870a488baf5a808adbfbc006]
I20250628 07:52:46.399937  2237 raft_consensus.cc:2270] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:46.400246  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7017192359304fa08eb3df34abc8c536 [id=5699b079b64a4864a611eb6bb02ed256]
I20250628 07:52:46.401409  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_be28b907172e4b36b346351fbc249f7a [id=575c9ff460e349f38b63afca848a22bb]
I20250628 07:52:46.402575  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_a90d3fd61b424016a14e3add816d2218 [id=57ab80d617474a998f26259768722a07]
I20250628 07:52:46.403285  1230 tablet_bootstrap.cc:492] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:46.403748  1230 ts_tablet_manager.cc:1397] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.236s	user 0.008s	sys 0.003s
I20250628 07:52:46.403774  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_123910cc7356492f93d0e1d1df124ad9 [id=5a391fa6a42f4b828a6eb4cc5e76465c]
I20250628 07:52:46.405777  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_8a1536aea0c64577a92168afbd496c5c [id=5bbb2a3c166044139f30f8650e698936]
I20250628 07:52:46.406090  1230 raft_consensus.cc:357] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.406934  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_18695aa15f7843b9a5388e863015aae0 [id=5be853f3c9e74f0a8cebb878f97e4f62]
I20250628 07:52:46.408133  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc]
I20250628 07:52:46.409260  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_949d6d04ea9e4eb4a7af04bd4d201718 [id=637c356534054cef9cdada8dfa2cf907]
I20250628 07:52:46.410387  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 [id=63b87421f31d454dbc6999067f297582]
I20250628 07:52:46.411597  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_5c036fde993047db95ee90d9ffd4be8d [id=640ae94271994216b7cbf2ffdef1cec9]
I20250628 07:52:46.412715  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_f6dc476a6f3b42afa13683cc9611a247 [id=6514b4f905774cf3bfec0ddc73804534]
I20250628 07:52:46.402063  2237 ts_tablet_manager.cc:1905] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:46.414984  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8]
I20250628 07:52:46.416062  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d86e7bc180df48c0b8358485d2a8bc2f [id=693a9f55cd4d4fcca72d0d27c3cc8b07]
I20250628 07:52:46.417188  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 [id=69980641f09443da87a0d85cb5cca3aa]
I20250628 07:52:46.418326  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_fe7551bb42dd452ab32b051075e3398b [id=6ad3b42bbcbd4d5e894480654bf6bebe]
I20250628 07:52:46.419461  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be]
I20250628 07:52:46.420639  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_0f6f46bde0ed463eb196dfdf0f049317 [id=6ceec3d9bfe142268afe00a134a83201]
I20250628 07:52:46.420835  1234 tablet_bootstrap.cc:654] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.422081  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 [id=6de11b06df364014af9b55fa276f3a7a]
I20250628 07:52:46.423218  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d]
I20250628 07:52:46.424316  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_bb662e3441d74b638d7c33dc69755650 [id=6e9ea1b8ad2d4977a835f5b0c03d1dc8]
I20250628 07:52:46.425505  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_6fd35d7894eb4e9891f539a62d743e62 [id=700083322b4a4b118855057493210fb7]
I20250628 07:52:46.425387  1229 consensus_queue.cc:260] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.427143  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3adf909f0f88430aa21e13480ae65522 [id=75edd41d3e954616b79662475b416941]
I20250628 07:52:46.427508  1229 ts_tablet_manager.cc:1428] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.052s	user 0.001s	sys 0.005s
I20250628 07:52:46.428288  1229 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:46.429706  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_5baf5e16d9f6482ab94b15a2706e3fbd [id=77ef1587831e44bd9a93a373b6fd9d0f]
I20250628 07:52:46.431591  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d31add8b03f14004af81e01854d4ace9 [id=79aa7a6bdf0f407f97ad9fc126ba6e8c]
I20250628 07:52:46.433930  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7a076a51572f4693af02750b6627265e [id=7b1235959a5c4e3b9d4c22edcbba14d6]
I20250628 07:52:46.436656  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c]
I20250628 07:52:46.438340  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de [id=82f7f172b18f4b59afcc216ff724acc3]
I20250628 07:52:46.439617  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 [id=839e2998d8e642f9a4fd0f7d376cefa4]
I20250628 07:52:46.442726  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6]
I20250628 07:52:46.443707  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_992ef1fbcdb840c382cc60a218166855 [id=868a11eb38d6484b98187acb9813872f]
I20250628 07:52:46.444825  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff]
I20250628 07:52:46.446522  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d2097dcb437041b98aef0c014b4e34e9 [id=89aacd4ca70d47efaf90b5662fe56b85]
I20250628 07:52:46.449705  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_5f97f65bd045458e83a0d56856db2b89 [id=8c576a9b2b00448fac7e76983bde42e6]
W20250628 07:52:46.451047   533 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:46.451512  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_f94575e525fe42959d7f99555b557ead [id=8ccc90997f7a4528ae77013e11610918]
I20250628 07:52:46.452869  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_4bed6916b6b14023b7099f15be8c1edc [id=8d3271dc7fa24639bdc1bf5d4104a459]
I20250628 07:52:46.454084  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_af46d3e348f94f0fbd27c3f97a2028ff [id=93aca27bae214b44a27421372584676f]
I20250628 07:52:46.455361  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]
I20250628 07:52:46.456597  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31]
I20250628 07:52:46.459101  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c8df75abf8c64fa4adc3102bb5f22ff2 [id=979223400058407baeaba004d05ab8ad]
I20250628 07:52:46.460336  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_ecc8fdb9f92341d38931d4b776b8114a [id=97a9ff9b1c5e4f808220992750b079a9]
I20250628 07:52:46.461601  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af]
I20250628 07:52:46.462877  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_59e9b18b772e4d5dafd78313d4b47800 [id=9a2c30b2d6a6409dbd3565a4b74ba210]
I20250628 07:52:46.464015  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_cce67d1f9bbb4434a6e80abe70297d1c [id=9af4a5a7e8aa43d6aa039ad94841acb7]
I20250628 07:52:46.466046  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_a011102f62ad4163af1c9aa371b6b29f [id=9c8759743f2c49cf911b12ca926da61f]
I20250628 07:52:46.467188  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2]
I20250628 07:52:46.468300  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 [id=9ed7f5db10a4467ea2551b27fd9a02d6]
I20250628 07:52:46.469499  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_de48ef8f76e3475a8ec0919c675b8711 [id=9f256f66bce3433d9f694f07c12964d1]
I20250628 07:52:46.470795  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 [id=a01aff122c7047bc8a2b83587776adea]
I20250628 07:52:46.472697  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_49b746aa19f841728b4cf05fa450c090 [id=a12bdfa9daa947988fdabac0b95e453a]
I20250628 07:52:46.473253  1230 raft_consensus.cc:383] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.474059  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_77748d06f2a54596a43e8a89b5b56011 [id=a32a0dbf0f3a44ff8703768610951a4a]
I20250628 07:52:46.474366  1230 raft_consensus.cc:738] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.475047  1230 consensus_queue.cc:260] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.476649  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_9ac8a7257de34accbdf87e0548648d25 [id=a70a1a3bd354432e9ebf6211b0a7a5b7]
I20250628 07:52:46.477151  2237 ts_tablet_manager.cc:1918] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:52:46.477452  1230 ts_tablet_manager.cc:1428] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.073s	user 0.005s	sys 0.000s
I20250628 07:52:46.477538  2237 log.cc:1199] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/f845f32699044867af6f0f2b655cad8e
I20250628 07:52:46.478027  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7c51004dd5724c6499326369c8712f8a [id=a7843107992b478b8e0b560eb2f13fa5]
I20250628 07:52:46.478180  2237 ts_tablet_manager.cc:1939] T f845f32699044867af6f0f2b655cad8e P 6005c44dfe4848f18a304621cd49e934: Deleting consensus metadata
I20250628 07:52:46.479794  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3f9fa575c08344b9965c9fb9bccc8c85 [id=a8e8d7b379564c7699f75b69b507bb73]
I20250628 07:52:46.481065  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f]
I20250628 07:52:46.482592  1229 tablet_bootstrap.cc:654] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.483594  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_04dc0144b3954979b9fdcce449c44782 [id=ad5a5ac2a4024b439c1d4fbe5e82509d]
I20250628 07:52:46.485541  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d74975b9cd2942c98865802e679ef4d5 [id=af0bd34359f84793a28be3c92a52612e]
I20250628 07:52:46.486706  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 [id=afffd13a151540f6863fc55f1a0c3f8c]
I20250628 07:52:46.487879  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_743f866ef4ae435180af5ecffcb04bc2 [id=b3a09493549c479499d0eb68d3841bc7]
I20250628 07:52:46.491096  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_ea4026116e0e482e84f4f48211e0079c [id=b91ffe1c2ac54b0f81f812bf9c13a641]
I20250628 07:52:46.492292  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c97313f05be54e4da2bc4cbbc04c8682 [id=b98b8985d5ad44b1a1618aa721533c5c]
I20250628 07:52:46.493526  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_b46689c124f54809961340f6c6b317d5 [id=bc2345b772a947e696627993d69e5e02]
I20250628 07:52:46.494717  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502]
I20250628 07:52:46.494846  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 444) took 4005 ms. Trace:
I20250628 07:52:46.495124  2237 rpcz_store.cc:276] 0628 07:52:42.489814 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.491336 (+  1522us) service_pool.cc:225] Handling call
0628 07:52:46.389718 (+3898382us) ts_tablet_manager.cc:1145] Deleting tablet f845f32699044867af6f0f2b655cad8e
0628 07:52:46.389794 (+    76us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.430026 (+ 40232us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:46.476927 (+ 46901us) tablet_metadata.cc:641] Metadata flushed
0628 07:52:46.494796 (+ 17869us) inbound_call.cc:173] Queueing success response
Metrics: {"spinlock_wait_cycles":2304,"tablet-delete.queue_time_us":3894392}
I20250628 07:52:46.496140  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 [id=beca5a78fa254e6eb8732613e47278fb]
I20250628 07:52:46.496186   532 catalog_manager.cc:4928] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): tablet f845f32699044867af6f0f2b655cad8e (table default.Table_92144695519f4cc9a21fedc487712d03 [id=77ccc4338ec448b6b07eace3de191874]) successfully deleted
I20250628 07:52:46.500226  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d65bc763056e4ce181baccde39c92ab9 [id=c3789b7fe8064911bd4095ae69bfd8e5]
I20250628 07:52:46.501518  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4]
I20250628 07:52:46.504392  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52]
I20250628 07:52:46.505606  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a]
I20250628 07:52:46.506369  1234 tablet_bootstrap.cc:492] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.506749  1234 ts_tablet_manager.cc:1397] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.123s	user 0.013s	sys 0.000s
I20250628 07:52:46.506832  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_8dcb758f32dc4345acc05a8e494ea755 [id=cc7adfe6bb724dc79d3fa8c5e4a24ded]
I20250628 07:52:46.507939  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_4ee15e08b0884908a81ad107f6738fe8 [id=cce2fcb4460e4fef8025a77e355c1890]
I20250628 07:52:46.509285  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a]
I20250628 07:52:46.511241  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960]
I20250628 07:52:46.512427  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_5d099833475e4b44b3fda7923e027a37 [id=d49f6e667b6e409c9f8f225158c16236]
I20250628 07:52:46.513621  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_4107fb91f5704f38b337a443ee54704c [id=d4a373f636634fd3ac4b34d1f31ca380]
I20250628 07:52:46.514827  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d34b80de26354403b4ea3b3994522090 [id=d535cd0c729146ff97bb42c7db33bbcb]
I20250628 07:52:46.516029  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_9e1de6bb51104538894960ac45e8b097 [id=d537b3af320d4a858ba125b23bfe0fa2]
I20250628 07:52:46.508756  1234 raft_consensus.cc:357] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.516727  1234 raft_consensus.cc:383] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.516973  1234 raft_consensus.cc:738] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.517463  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e10aee64839f49a18fc40406bb808709 [id=d607ae35fec24cab9d45e70c9ec351f3]
I20250628 07:52:46.517638  1234 consensus_queue.cc:260] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.519405  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3cf97e46be3049c0b1faa4e17cfa70eb [id=d8e81f13937f4fde8192b2293190c950]
I20250628 07:52:46.519522  1234 ts_tablet_manager.cc:1428] T d9e0ea5e300347c7adc8e30b9070976b P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.002s
I20250628 07:52:46.520344  1234 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.521670  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_061d0f52cd51486a88f1e59480dffc7f [id=da004477b85348e69492c13b6404eed4]
I20250628 07:52:46.523816  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_09dfad9563454a0e9624d7e15961a374 [id=dce9747fa94f4f86bdafae93dbf53339]
I20250628 07:52:46.524893  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d3053995fff94245ae716d9ea4118068 [id=e1a9ab2b162b43399279afd19dd0f093]
I20250628 07:52:46.526677  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d9f1fae1f46f45efbde92d35775fdd0e [id=e32e5fdf4ddb42d794ebe9d70ad300fb]
I20250628 07:52:46.527904  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3eb1f00557094612af6b3582a0ce2679 [id=e491563484064b93b14fe4b3a15c6ebd]
I20250628 07:52:46.529512  2715 raft_consensus.cc:491] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.530014  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b]
I20250628 07:52:46.529934  2715 raft_consensus.cc:513] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.531440  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43]
I20250628 07:52:46.532627  2715 leader_election.cc:290] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.532699  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_d8ead90f917d46b9beef6badf9823be1 [id=e658c2419bd748e98d19b83528587ca7]
I20250628 07:52:46.533936  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_e20c49ea9aae4980b56ee14957a8c928 [id=e7e6cd0579cf44d7877364fcb5c21e28]
I20250628 07:52:46.533877   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:46.534993   920 leader_election.cc:343] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.537804  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_753078c239dc43a6a5125732a2ff887c [id=eeec5abe55af475f8195a9ec1880ca62]
I20250628 07:52:46.540808  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 445) took 3775 ms. Trace:
I20250628 07:52:46.540962  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_c974994f851542aa92291ccc82d531e0 [id=f7cf0e3d20f645d2ad5d9b5ee3637834]
I20250628 07:52:46.542327  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_db0904127a6144d7acb537f17d6891d6 [id=f8b5a176637544b0b21ec14c5bb5598f]
I20250628 07:52:46.543499  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_88c945c579fe4aaf9c7acecfc6b88288 [id=f9005880df6b41aa86533a453b679f3d]
I20250628 07:52:46.544286  2658 raft_consensus.cc:491] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.544757  2658 raft_consensus.cc:513] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.545583  2740 catalog_manager.cc:671] Loaded metadata for table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312]
I20250628 07:52:46.545429  2237 rpcz_store.cc:276] 0628 07:52:42.765115 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.765214 (+    99us) service_pool.cc:225] Handling call
0628 07:52:46.540442 (+3775228us) ts_tablet_manager.cc:1145] Deleting tablet 88d4a84c8ed7475c9bf043aaba8ad624
0628 07:52:46.540506 (+    64us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.540757 (+   251us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3760267}
W20250628 07:52:46.546439   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 88d4a84c8ed7475c9bf043aaba8ad624 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 88d4a84c8ed7475c9bf043aaba8ad624 already in progress: creating tablet
I20250628 07:52:46.546612  2658 leader_election.cc:290] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061)
I20250628 07:52:46.548537   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5febe562643048f083e579c5f7ba0793" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:46.548975   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5febe562643048f083e579c5f7ba0793" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:46.550328  1062 leader_election.cc:343] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.550709  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 452) took 2938 ms. Trace:
W20250628 07:52:46.550892  1062 leader_election.cc:343] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.551216  1062 leader_election.cc:304] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:46.552119  2658 raft_consensus.cc:2747] T 5febe562643048f083e579c5f7ba0793 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:46.555761  1230 tablet_bootstrap.cc:492] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:46.551074  2237 rpcz_store.cc:276] 0628 07:52:43.610995 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.611098 (+   103us) service_pool.cc:225] Handling call
0628 07:52:46.549519 (+2938421us) ts_tablet_manager.cc:1145] Deleting tablet 51a44ec161684f74b228830ff06f1957
0628 07:52:46.549586 (+    67us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.549819 (+   233us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2937906}
W20250628 07:52:46.564100   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 51a44ec161684f74b228830ff06f1957 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 51a44ec161684f74b228830ff06f1957 already in progress: creating tablet
I20250628 07:52:46.564723  1234 tablet_bootstrap.cc:654] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.568534  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 460) took 2605 ms. Trace:
I20250628 07:52:46.568840  2237 rpcz_store.cc:276] 0628 07:52:43.963600 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.963697 (+    97us) service_pool.cc:225] Handling call
0628 07:52:46.568194 (+2604497us) ts_tablet_manager.cc:1145] Deleting tablet 670f91ee94f7460caa0790e232ac7fa8
0628 07:52:46.568241 (+    47us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.568486 (+   245us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2603880}
W20250628 07:52:46.578128   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 670f91ee94f7460caa0790e232ac7fa8 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 670f91ee94f7460caa0790e232ac7fa8 already in progress: creating tablet
I20250628 07:52:46.580185  1230 tablet_bootstrap.cc:654] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.581211  2237 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:41482 (request call id 459) took 2617 ms. Trace:
I20250628 07:52:46.581529  2237 rpcz_store.cc:276] 0628 07:52:43.963361 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.965991 (+  2630us) service_pool.cc:225] Handling call
0628 07:52:46.580826 (+2614835us) ts_tablet_manager.cc:1145] Deleting tablet 6ce66199d1614826bf618819f6928383
0628 07:52:46.580876 (+    50us) ts_tablet_manager.cc:1065] Acquired tablet manager lock
0628 07:52:46.581159 (+   283us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2614377}
W20250628 07:52:46.582408   532 catalog_manager.cc:4908] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): delete failed for tablet 6ce66199d1614826bf618819f6928383 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 6ce66199d1614826bf618819f6928383 already in progress: creating tablet
I20250628 07:52:46.585620  1229 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:46.586045  1229 ts_tablet_manager.cc:1397] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.158s	user 0.009s	sys 0.000s
I20250628 07:52:46.588357  2740 tablet_loader.cc:96] loaded metadata for tablet 01b1767597424753a588090e9228d133 (table default.Table_3eb1f00557094612af6b3582a0ce2679 [id=e491563484064b93b14fe4b3a15c6ebd])
I20250628 07:52:46.589370  1229 raft_consensus.cc:357] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.590420  1229 raft_consensus.cc:383] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.590633  2740 tablet_loader.cc:96] loaded metadata for tablet 033bea0d32c048e29df6cc3443fc106a (table default.Table_14ad18ef5e224208bffdbc4a88018111 [id=4c9749c443364cee95d905b8c5973e3e])
I20250628 07:52:46.590777  1229 raft_consensus.cc:738] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.591704  2740 tablet_loader.cc:96] loaded metadata for tablet 067cf8a0fc764d5ab29a5a80e62a5ac7 (table default.Table_ecc8fdb9f92341d38931d4b776b8114a [id=97a9ff9b1c5e4f808220992750b079a9])
I20250628 07:52:46.591476  1229 consensus_queue.cc:260] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.593565  2740 tablet_loader.cc:96] loaded metadata for tablet 08afbaee7cb74ce8a6b2958a6828eca1 (table default.Table_d2097dcb437041b98aef0c014b4e34e9 [id=89aacd4ca70d47efaf90b5662fe56b85])
I20250628 07:52:46.594146  1234 tablet_bootstrap.cc:492] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.594532  1234 ts_tablet_manager.cc:1397] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.074s	user 0.009s	sys 0.000s
I20250628 07:52:46.595522  2740 tablet_loader.cc:96] loaded metadata for tablet 0b251095244548629ae93d52e6c80ccc (table default.Table_cce67d1f9bbb4434a6e80abe70297d1c [id=9af4a5a7e8aa43d6aa039ad94841acb7])
I20250628 07:52:46.596609  2740 tablet_loader.cc:96] loaded metadata for tablet 0bdb4c499be441379663830f26d80204 (table default.Table_1b3d4ecf8a5f4aae87489ab6b1c5e05c [id=48a7ea3af0174a36af72c55321ef8a3e])
I20250628 07:52:46.597713  2740 tablet_loader.cc:96] loaded metadata for tablet 0c4a31f42b454a0b9dfac82be51563a8 (table default.Table_a5aaafa5f2c84037b58a9040e19581dd [id=390ca1dfaf6a44b193876507a9dbe929])
I20250628 07:52:46.598500  1234 raft_consensus.cc:357] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.599090  1234 raft_consensus.cc:383] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.599327  1234 raft_consensus.cc:738] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.601071  2740 tablet_loader.cc:96] loaded metadata for tablet 0d0c851c1efb4774a834b0662b092f6c (table default.Table_d3fc569c0e854f788cd8355607639d3a [id=12c2d9bf9bc8422a9f415382c8202daa])
I20250628 07:52:46.602510  1127 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.602875  2740 tablet_loader.cc:96] loaded metadata for tablet 0e96d996085d4091b966ee51b02c140b (table default.Table_577ffb94469047e2a14ccb8d89c9a4d6 [id=3e63dedd51ba4d36bd5f7d53a2dd94e4])
I20250628 07:52:46.603072  1127 raft_consensus.cc:2466] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:46.604509   921 leader_election.cc:304] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:46.605490  2699 raft_consensus.cc:2802] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:46.605809  2699 raft_consensus.cc:491] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:46.606155  2699 raft_consensus.cc:3058] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.610878  2740 tablet_loader.cc:96] loaded metadata for tablet 1160beb0d6c34e12b1f95d6fbdd6ab5f (table default.Table_d34b80de26354403b4ea3b3994522090 [id=d535cd0c729146ff97bb42c7db33bbcb])
I20250628 07:52:46.610574  1234 consensus_queue.cc:260] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.611937  2740 tablet_loader.cc:96] loaded metadata for tablet 12d856ea75ef4f558efc58afd73339f4 (table default.Table_8579b9af0fe6471c99b0ab6cebc7ed29 [id=6de11b06df364014af9b55fa276f3a7a])
I20250628 07:52:46.612948  2740 tablet_loader.cc:96] loaded metadata for tablet 132cd82d39e84e85a37f21ae208fd165 (table default.Table_7017192359304fa08eb3df34abc8c536 [id=5699b079b64a4864a611eb6bb02ed256])
I20250628 07:52:46.614048  2740 tablet_loader.cc:96] loaded metadata for tablet 14fedcf110e64c33978004743c5096ed (table default.Table_17cd0eb6b1c84e65bccdd4caf5e41103 [id=63b87421f31d454dbc6999067f297582])
I20250628 07:52:46.615108  2740 tablet_loader.cc:96] loaded metadata for tablet 15dd5b509d5c43d68ff6ecc960aa9c6e (table default.Table_743f866ef4ae435180af5ecffcb04bc2 [id=b3a09493549c479499d0eb68d3841bc7])
I20250628 07:52:46.617542  2740 tablet_loader.cc:96] loaded metadata for tablet 174cab60958f41759967ffe9dda27991 (table default.Table_b63e7c6a006d4e12b58a66a5ef9615b4 [id=2609d3d6796f4fa0b9cb750fa925da21])
I20250628 07:52:46.617647  1234 ts_tablet_manager.cc:1428] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.023s	user 0.006s	sys 0.000s
I20250628 07:52:46.617770  2699 raft_consensus.cc:513] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.618608  2740 tablet_loader.cc:96] loaded metadata for tablet 181c57eeb50745beb09c36a38b75d9be (table default.Table_c81c2135bc5a4417b706d1c042207cc0 [id=0397ad0e0e114f359480497eac5e7712])
I20250628 07:52:46.619647  2740 tablet_loader.cc:96] loaded metadata for tablet 19e7760b95b94336a985cdd128cef8de (table default.Table_c6c0d1b4e3f549e6a1f103267d886ee9 [id=9ed7f5db10a4467ea2551b27fd9a02d6])
I20250628 07:52:46.620137   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
W20250628 07:52:46.621274   920 leader_election.cc:343] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.622172  1116 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8a0f311c619443198e33dfef20a3f4a4" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:46.622682  1116 raft_consensus.cc:3058] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.624266  2740 tablet_loader.cc:96] loaded metadata for tablet 1ed24dbf9c234b43b352114d5219bbea (table default.Table_88c945c579fe4aaf9c7acecfc6b88288 [id=f9005880df6b41aa86533a453b679f3d])
I20250628 07:52:46.625295  2740 tablet_loader.cc:96] loaded metadata for tablet 1efec32bd8cd468eb13825d51722f7cc (table default.Table_2780965e7e5d4d528b1d63995873222f [id=127fda336df94c48a181d1a59e0d2f00])
I20250628 07:52:46.629211  2740 tablet_loader.cc:96] loaded metadata for tablet 20acb22449ac4203bbed31067df0c553 (table default.Table_e20c49ea9aae4980b56ee14957a8c928 [id=e7e6cd0579cf44d7877364fcb5c21e28])
I20250628 07:52:46.629765  1229 ts_tablet_manager.cc:1428] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.043s	user 0.005s	sys 0.000s
I20250628 07:52:46.630540  2740 tablet_loader.cc:96] loaded metadata for tablet 21ee5fbf572c4530939c1c103b89737e (table default.Table_e393a3a8856046f1a8d11d7ab8c485ee [id=3311e522b275451fa31744d77fcc8654])
I20250628 07:52:46.632222  2740 tablet_loader.cc:96] loaded metadata for tablet 24625050151f4b50aa8349c35184570a (table default.Table_3cf97e46be3049c0b1faa4e17cfa70eb [id=d8e81f13937f4fde8192b2293190c950])
I20250628 07:52:46.633255  2740 tablet_loader.cc:96] loaded metadata for tablet 2595308cbc0d44f0abfd0a126f16d4df (table default.Table_bb6804bddac34e8db4751977623c7ab8 [id=28517a67c8e24b8e9575c49c90edea15])
I20250628 07:52:46.638383  2740 tablet_loader.cc:96] loaded metadata for tablet 2c932ba2738a414da4405677d886dabe (table default.Table_d31add8b03f14004af81e01854d4ace9 [id=79aa7a6bdf0f407f97ad9fc126ba6e8c])
I20250628 07:52:46.641568  2740 tablet_loader.cc:96] loaded metadata for tablet 2f8099bd0c9649b492edd8b657fd19e3 (table default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c])
I20250628 07:52:46.643332  2740 tablet_loader.cc:96] loaded metadata for tablet 33ae7e653d644224a07a3bf29ef9fcd9 (table default.Table_949d6d04ea9e4eb4a7af04bd4d201718 [id=637c356534054cef9cdada8dfa2cf907])
I20250628 07:52:46.644387  2740 tablet_loader.cc:96] loaded metadata for tablet 340b3ef1c06d48bc882e16e5765d4e22 (table default.Table_57d70b50d59c4f34bc68e39e92d7a6ef [id=50bb32575d534f1c81028fa68756d288])
I20250628 07:52:46.646092  2740 tablet_loader.cc:96] loaded metadata for tablet 35fcce92db9a4e4a81f9d9db9d6283b8 (table default.Table_1cf78a5cb3c5490bae2317600c643635 [id=2613a92dda0d4236b4a4422cf612a6c6])
I20250628 07:52:46.649083  2740 tablet_loader.cc:96] loaded metadata for tablet 36eebc4a46bd454da5bc176df2a9680f (table default.Table_d74975b9cd2942c98865802e679ef4d5 [id=af0bd34359f84793a28be3c92a52612e])
I20250628 07:52:46.650161  2740 tablet_loader.cc:96] loaded metadata for tablet 39ba8057357241b99882e883f6af93e7 (table default.Table_de48ef8f76e3475a8ec0919c675b8711 [id=9f256f66bce3433d9f694f07c12964d1])
I20250628 07:52:46.650862  1116 raft_consensus.cc:2466] T 8a0f311c619443198e33dfef20a3f4a4 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:46.651283  2740 tablet_loader.cc:96] loaded metadata for tablet 3cfb8eefaa45432bb50cc729a401a2ec (table default.Table_4ee15e08b0884908a81ad107f6738fe8 [id=cce2fcb4460e4fef8025a77e355c1890])
I20250628 07:52:46.652927  2740 tablet_loader.cc:96] loaded metadata for tablet 460693e542d7452c9f80777dbff67b57 (table default.Table_e04e4751022f4e2fbc686f49b598150e [id=14de477c143d4d2084393ef5d658e4d0])
I20250628 07:52:46.653965  2740 tablet_loader.cc:96] loaded metadata for tablet 46e0aa090f544e8aa0a426c61925f345 (table default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8])
I20250628 07:52:46.655155  2699 leader_election.cc:290] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.655745  1234 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.657042  2740 tablet_loader.cc:96] loaded metadata for tablet 48b376ff0fbc47b086870e1503bd2d20 (table default.Table_0f6f46bde0ed463eb196dfdf0f049317 [id=6ceec3d9bfe142268afe00a134a83201])
I20250628 07:52:46.658123  2740 tablet_loader.cc:96] loaded metadata for tablet 4a2409a1155441c89df8d04722cd35ed (table default.Table_c8bc72fa0f974cfebafe4cc4b90bb008 [id=a01aff122c7047bc8a2b83587776adea])
I20250628 07:52:46.659149  2740 tablet_loader.cc:96] loaded metadata for tablet 4ba7d2634cb74f6f858a5a3db8493b6a (table default.Table_7c51004dd5724c6499326369c8712f8a [id=a7843107992b478b8e0b560eb2f13fa5])
I20250628 07:52:46.660197  2740 tablet_loader.cc:96] loaded metadata for tablet 4c2e91519c6b4b509d96f508546c8fc4 (table default.Table_7437789ae57b4f029d85e0722aa8fa84 [id=442671f219f54e26b6f9d8958301fbd8])
I20250628 07:52:46.661130  1234 tablet_bootstrap.cc:654] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.661866  2740 tablet_loader.cc:96] loaded metadata for tablet 4d21243664ea4bbeb4a7c31d8c019486 (table default.Table_8a1536aea0c64577a92168afbd496c5c [id=5bbb2a3c166044139f30f8650e698936])
I20250628 07:52:46.663748  2740 tablet_loader.cc:96] loaded metadata for tablet 4eac11942d104acab461d483252128c5 (table default.Table_cf53da78d4564be08dd5673e1b5a895b [id=6e10b2ebb2324d7198325e018813413d])
I20250628 07:52:46.667479   921 leader_election.cc:304] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:46.668577  2699 raft_consensus.cc:2802] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:46.669054  2699 raft_consensus.cc:695] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:46.669860  2699 consensus_queue.cc:237] T 8a0f311c619443198e33dfef20a3f4a4 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.674512  1229 tablet_bootstrap.cc:492] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:46.686549  2740 tablet_loader.cc:96] loaded metadata for tablet 5393c4c8cac1492da21b3e366aa8eaf7 (table default.Table_f94575e525fe42959d7f99555b557ead [id=8ccc90997f7a4528ae77013e11610918])
I20250628 07:52:46.688254  2740 tablet_loader.cc:96] loaded metadata for tablet 547d0e51544c49fcbcfdcd05f2390025 (table default.Table_f5b1b57d6e754cccb6eb0a1405289fde [id=53bd74ded2bf4f5fa0edb44f63d4a340])
I20250628 07:52:46.694301  2657 raft_consensus.cc:491] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.694696  2657 raft_consensus.cc:513] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.698019  2440 raft_consensus.cc:491] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.698511  2440 raft_consensus.cc:513] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.699995  1234 tablet_bootstrap.cc:492] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.700372  1234 ts_tablet_manager.cc:1397] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.045s	user 0.007s	sys 0.003s
I20250628 07:52:46.701453  2440 leader_election.cc:290] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.702965   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:46.703631   984 raft_consensus.cc:2391] T a353cc9eedf34cc7aa4cfbcc7198247b P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:46.702664  1234 raft_consensus.cc:357] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.704555  1234 raft_consensus.cc:383] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.704825  1234 raft_consensus.cc:738] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.704687  1116 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a353cc9eedf34cc7aa4cfbcc7198247b" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.705423  1116 raft_consensus.cc:2391] T a353cc9eedf34cc7aa4cfbcc7198247b P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 6a58514e01ea4989809bc2992fb2b806 in this term.
I20250628 07:52:46.706060  1234 consensus_queue.cc:260] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.708495  2657 leader_election.cc:290] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.707314   757 leader_election.cc:304] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:46.709165   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934" is_pre_election: true
W20250628 07:52:46.712617   920 leader_election.cc:343] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.714890  2740 tablet_loader.cc:96] loaded metadata for tablet 55045da381224ff28ae934ac3e28dd05 (table default.Table_7ac89842d7e54a2ea03d7f64f4a92175 [id=28124975e8e246c68bd570b13fa5af2a])
I20250628 07:52:46.715324  2660 raft_consensus.cc:3058] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.716070  2740 tablet_loader.cc:96] loaded metadata for tablet 586f1b7dd30e4998abebaa1fb2983e2b (table default.Table_21bdd4d912164bcbaf63188ced2ce8ad [id=883a1b6945714d61b674bbdbf8aa5cff])
I20250628 07:52:46.717098  2740 tablet_loader.cc:96] loaded metadata for tablet 58d853d69a5145cdabb33437463d4aa2 (table default.Table_948ee6f5274e4638b253dcf9f5df2e31 [id=4454ad1ffc2744a1b55d3772f0722258])
I20250628 07:52:46.718168  2740 tablet_loader.cc:96] loaded metadata for tablet 58e7ec1f221f41ca8f6b019faeec3377 (table default.Table_59e9b18b772e4d5dafd78313d4b47800 [id=9a2c30b2d6a6409dbd3565a4b74ba210])
I20250628 07:52:46.718832  1116 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.719256  2740 tablet_loader.cc:96] loaded metadata for tablet 5aece66627754857a0fc128c72269ea9 (table default.Table_be28b907172e4b36b346351fbc249f7a [id=575c9ff460e349f38b63afca848a22bb])
I20250628 07:52:46.719421  1116 raft_consensus.cc:2466] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:46.720320  2740 tablet_loader.cc:96] loaded metadata for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844])
I20250628 07:52:46.720327   921 leader_election.cc:304] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:46.721920  2715 raft_consensus.cc:2802] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:46.722235  2715 raft_consensus.cc:491] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:46.722576  2715 raft_consensus.cc:3058] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.722622  2660 raft_consensus.cc:2747] T a353cc9eedf34cc7aa4cfbcc7198247b P 6005c44dfe4848f18a304621cd49e934 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:46.726135  1234 ts_tablet_manager.cc:1428] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.026s	user 0.006s	sys 0.001s
I20250628 07:52:46.731564  2740 tablet_loader.cc:96] loaded metadata for tablet 5d5b34ea6d7740aca2fc9827f1fefc37 (table default.Table_af46d3e348f94f0fbd27c3f97a2028ff [id=93aca27bae214b44a27421372584676f])
I20250628 07:52:46.734591  2740 tablet_loader.cc:96] loaded metadata for tablet 5f5d8a873a9f4197b9d5304ae08c12f3 (table default.Table_1ed469ca3b6a4ae8810fd9dc3da570bf [id=222109d2472a4f079bc6e4abf9423ee6])
I20250628 07:52:46.735934  2740 tablet_loader.cc:96] loaded metadata for tablet 5f77a64b8e054f45acbad44d02af0e82 (table default.Table_9775925de3b34654b11d221bc66cb7fd [id=1f4b61bbe4284a718e2adc4e1a1fcc59])
I20250628 07:52:46.736964  1234 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.750638  2715 raft_consensus.cc:513] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.751324  2142 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 4 FOLLOWER]: Refusing update from remote peer 992ac4c27fb744deb1edf7765553ef08: Log matching property violated. Preceding OpId in replica: term: 3 index: 417. Preceding OpId from leader: term: 4 index: 556. (index mismatch)
I20250628 07:52:46.753140   815 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6005c44dfe4848f18a304621cd49e934"
I20250628 07:52:46.753717  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a5ac50e627c0470782f96bd77a61f763" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:46.754252  1121 raft_consensus.cc:3058] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:52:46.754335   920 leader_election.cc:343] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:46.758585  2761 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 556, Last known committed idx: 417, Time since last communication: 0.000s
I20250628 07:52:46.762787  1121 raft_consensus.cc:2466] T a5ac50e627c0470782f96bd77a61f763 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:46.766717  1234 tablet_bootstrap.cc:654] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.769824  1229 tablet_bootstrap.cc:654] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:46.771160  2740 tablet_loader.cc:96] loaded metadata for tablet 5febe562643048f083e579c5f7ba0793 (table default.Table_c8df75abf8c64fa4adc3102bb5f22ff2 [id=979223400058407baeaba004d05ab8ad])
I20250628 07:52:46.775902  2715 leader_election.cc:290] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.779505   555 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Refusing update from remote peer 992ac4c27fb744deb1edf7765553ef08: Log matching property violated. Preceding OpId in replica: term: 3 index: 555. Preceding OpId from leader: term: 4 index: 556. (index mismatch)
I20250628 07:52:46.781036   921 leader_election.cc:304] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 6005c44dfe4848f18a304621cd49e934
I20250628 07:52:46.782239  2762 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Connected to new peer: Peer: permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 556, Last known committed idx: 555, Time since last communication: 0.000s
I20250628 07:52:46.782435  2715 raft_consensus.cc:2802] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:46.783258  2715 raft_consensus.cc:695] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
I20250628 07:52:46.784504  2715 consensus_queue.cc:237] T a5ac50e627c0470782f96bd77a61f763 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.786064   815 raft_consensus.cc:3058] T 613eb4aafd7b4a93b0f2a9699a942f34 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.789647  2740 tablet_loader.cc:96] loaded metadata for tablet 60179e6765b145adbf637a8435eef724 (table default.Table_c47ff4f5324245c3b4b504c56d36649b [id=044b1a528f234b7c97a6f7066ad8169b])
I20250628 07:52:46.792883   538 catalog_manager.cc:5582] T f8bce65d32e042ab89ebe168d9657466 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:46.795554  2657 consensus_queue.cc:1035] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:46.791738  2740 tablet_loader.cc:96] loaded metadata for tablet 613eb4aafd7b4a93b0f2a9699a942f34 (table default.Table_9e1de6bb51104538894960ac45e8b097 [id=d537b3af320d4a858ba125b23bfe0fa2])
I20250628 07:52:46.799507  2740 tablet_loader.cc:96] loaded metadata for tablet 64f5ca605ce4480da9eaad08456ddb9a (table default.Table_d86e7bc180df48c0b8358485d2a8bc2f [id=693a9f55cd4d4fcca72d0d27c3cc8b07])
I20250628 07:52:46.805776  2440 raft_consensus.cc:491] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.806200  2440 raft_consensus.cc:513] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.808317  2440 leader_election.cc:290] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:46.810595  1129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:46.811241  1129 raft_consensus.cc:2391] T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6005c44dfe4848f18a304621cd49e934 in current term 1: Already voted for candidate 82f43cdfab544b17a6a71033983e65d4 in this term.
I20250628 07:52:46.813150   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "f9a553c69c254fdcaf413b53018bbfb2" candidate_uuid: "6005c44dfe4848f18a304621cd49e934" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:46.814939   757 leader_election.cc:304] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:46.817800  2440 raft_consensus.cc:3058] T f9a553c69c254fdcaf413b53018bbfb2 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:46.818266  1234 tablet_bootstrap.cc:492] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934: No bootstrap required, opened a new log
I20250628 07:52:46.818686  1234 ts_tablet_manager.cc:1397] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934: Time spent bootstrapping tablet: real 0.082s	user 0.008s	sys 0.002s
I20250628 07:52:46.806020   538 catalog_manager.cc:5582] T 5f77a64b8e054f45acbad44d02af0e82 P 82f43cdfab544b17a6a71033983e65d4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130). New cstate: current_term: 1 leader_uuid: "82f43cdfab544b17a6a71033983e65d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } health_report { overall_health: UNKNOWN } } }
I20250628 07:52:46.820976  1234 raft_consensus.cc:357] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.821589  1234 raft_consensus.cc:383] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.821813  1234 raft_consensus.cc:738] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6005c44dfe4848f18a304621cd49e934, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.828305   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet 5cacde13bc29458da81f29fb4fa2945a (table default.Table_60b7b090ea554a1ea11dd815c9b552ee [id=93f9610f0fac41f3b8f407a3efd9b844]). Expected version 1 got 0
I20250628 07:52:46.823457  2740 tablet_loader.cc:96] loaded metadata for tablet 661d92febf1142f99815f70ae9b27edc (table default.Table_ae262f1ee79148c4a8afbabd65d9ac89 [id=9466fec47b0c4c149be47db4afeb5b31])
I20250628 07:52:46.837937   538 catalog_manager.cc:5674] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061) does not have the latest schema for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937]). Expected version 2 got 0
I20250628 07:52:46.844957  2740 tablet_loader.cc:96] loaded metadata for tablet 67cd7523a7c74a278a1e2cd8dfeab2cd (table default.Table_c35cbc8840ec40728f62ffc4fb20ce69 [id=41902d471a304c5d985c48efbe81acd0])
I20250628 07:52:46.847627  2657 consensus_queue.cc:1035] T 613eb4aafd7b4a93b0f2a9699a942f34 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:52:46.848067  1234 consensus_queue.cc:260] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.850193  1234 ts_tablet_manager.cc:1428] T a5ac50e627c0470782f96bd77a61f763 P 6005c44dfe4848f18a304621cd49e934: Time spent starting tablet: real 0.031s	user 0.003s	sys 0.002s
I20250628 07:52:46.851127  1234 tablet_bootstrap.cc:492] T 8a0f311c619443198e33dfef20a3f4a4 P 6005c44dfe4848f18a304621cd49e934: Bootstrap starting.
I20250628 07:52:46.855360  2740 tablet_loader.cc:96] loaded metadata for tablet 6fdda1e4f7af468982a813f1ac023e01 (table default.Table_a211a91358794a51a65e048e19371379 [id=c9e6ae6d71a04ae99e69992a0b198c52])
I20250628 07:52:46.858047   541 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: Sending DeleteTablet for 3 replicas of tablet 7e2ecd14e8524f018dd3c0aecd564218
I20250628 07:52:46.859345   541 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:47964 (request call id 460) took 4460 ms. Trace:
I20250628 07:52:46.874188   951 tablet_service.cc:1515] Processing DeleteTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:35848
I20250628 07:52:46.874383   789 tablet_service.cc:1515] Processing DeleteTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:41482
I20250628 07:52:46.876195  2660 raft_consensus.cc:491] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20250628 07:52:46.876763   540 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.876631  2660 raft_consensus.cc:513] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:46.880043   757 leader_election.cc:343] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Not found: Tablet not found: fd67d7d9ba9446c19f1711fb5bd73ebd
I20250628 07:52:46.882539  2660 leader_election.cc:290] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
W20250628 07:52:46.883558   547 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.884404  2765 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.884805  2765 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:46.885646  2237 tablet_replica.cc:331] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934: stopping tablet replica
W20250628 07:52:46.886066   758 leader_election.cc:343] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): Not found: Tablet not found: fd67d7d9ba9446c19f1711fb5bd73ebd
I20250628 07:52:46.886415  2237 raft_consensus.cc:2241] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Raft consensus shutting down.
I20250628 07:52:46.886476   758 leader_election.cc:304] T fd67d7d9ba9446c19f1711fb5bd73ebd P 6005c44dfe4848f18a304621cd49e934 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6005c44dfe4848f18a304621cd49e934; no voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:46.886951  2237 raft_consensus.cc:2270] T 7e2ecd14e8524f018dd3c0aecd564218 P 6005c44dfe4848f18a304621cd49e934 [term 2 FOLLOWER]: Raft consensus is shut down!
I20250628 07:52:46.859623   541 rpcz_store.cc:276] 0628 07:52:42.398667 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.459042 (+ 60375us) service_pool.cc:225] Handling call
0628 07:52:42.472051 (+ 13009us) spinlock_profiling.cc:243] Waited 12.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea941046 00007f5be86601a4 00007f5be8660132 00007f5be8660061 00007f5be865ffdc 00007f5be865f292 00007f5be6957dcc 00007f5be6957206
0628 07:52:42.473216 (+  1165us) catalog_manager.cc:2368] Looking up, locking, and authorizing table
0628 07:52:42.473955 (+   739us) catalog_manager.cc:2790] Modifying in-memory table state
0628 07:52:42.474066 (+   111us) catalog_manager.cc:2799] Locking tablets
0628 07:52:42.474763 (+   697us) catalog_manager.cc:2813] Removing table and tablets from system table
0628 07:52:42.809311 (+334548us) catalog_manager.cc:2831] Removing table from by-name map
0628 07:52:42.809536 (+   225us) catalog_manager.cc:2847] Committing in-memory state
0628 07:52:46.857956 (+4048420us) catalog_manager.cc:2851] Aborting table tasks
0628 07:52:46.859283 (+  1327us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:42.731512 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:42.731937 (+   425us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:42.731964 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:42.731976 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:42.732310 (+   334us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:42.732349 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:42.732377 (+    28us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:42.732646 (+   269us) tablet.cc:699] Row locks acquired
0628 07:52:42.732658 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:42.732770 (+   112us) write_op.cc:270] Start()
0628 07:52:42.732858 (+    88us) write_op.cc:276] Timestamp: P: 1751097162732694 usec, L: 0
0628 07:52:42.732873 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:42.733409 (+   536us) log.cc:844] Serialized 765 byte log entry
0628 07:52:42.805347 (+ 71938us) op_driver.cc:464] REPLICATION: finished
0628 07:52:42.807014 (+  1667us) write_op.cc:301] APPLY: starting
0628 07:52:42.807086 (+    72us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:42.807195 (+   109us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:42.807287 (+    92us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:42.807565 (+   278us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:42.807708 (+   143us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:42.807750 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:42.807924 (+   174us) log.cc:844] Serialized 29 byte log entry
0628 07:52:42.808056 (+   132us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:42.808122 (+    66us) write_op.cc:454] Released schema lock
0628 07:52:42.808152 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:42.816285 (+  8133us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:42.816311 (+    26us) write_op.cc:454] Released schema lock
Metrics: {"spinlock_wait_cycles":28124032,"child_traces":[["op",{"apply.queue_time_us":1521,"apply.run_cpu_time_us":1354,"apply.run_wall_time_us":1354,"num_ops":2,"prepare.queue_time_us":255090,"prepare.run_cpu_time_us":2279,"prepare.run_wall_time_us":84762,"raft.queue_time_us":106,"raft.run_cpu_time_us":691,"raft.run_wall_time_us":691,"replication_time_us":72376}]]}
I20250628 07:52:46.889591   430 master-stress-test.cc:579] Killed test thread 8/10
I20250628 07:52:46.889844   430 master-stress-test.cc:577] Killing test thread 9/10
I20250628 07:52:46.889703  2142 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:46.890151   430 master-stress-test.cc:579] Killed test thread 9/10
I20250628 07:52:46.890362   430 master-stress-test.cc:577] Killing test thread 10/10
I20250628 07:52:46.890275  2142 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 023ed4cb582a44509bccf42b52c494d7 in term 4.
I20250628 07:52:46.890678   430 master-stress-test.cc:579] Killed test thread 10/10
I20250628 07:52:46.891043   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 708
I20250628 07:52:46.891340   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 
I20250628 07:52:46.893668  2769 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Leader pre-election won for term 5
W20250628 07:52:46.894528   545 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
W20250628 07:52:46.900372   545 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
W20250628 07:52:46.902513   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
W20250628 07:52:46.905229   546 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.905431  2769 raft_consensus.cc:491] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:46.905776  1230 tablet_bootstrap.cc:492] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:46.905814  2769 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 4 FOLLOWER]: Advancing to term 5
I20250628 07:52:46.906180  1230 ts_tablet_manager.cc:1397] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.351s	user 0.007s	sys 0.002s
W20250628 07:52:46.907548   546 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.907577  2768 tablet_replica.cc:331] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: stopping tablet replica
I20250628 07:52:46.908326  1230 raft_consensus.cc:357] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.908638  2768 raft_consensus.cc:2241] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 LEADER]: Raft consensus shutting down.
W20250628 07:52:46.905172   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.909582  2768 raft_consensus.cc:2270] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4 [term 2 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:46.911173   543 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.911548  1229 tablet_bootstrap.cc:492] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:46.911757  2768 ts_tablet_manager.cc:1905] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:46.912508  1229 ts_tablet_manager.cc:1397] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.238s	user 0.007s	sys 0.003s
W20250628 07:52:46.912820   541 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 72) took 1061 ms (client timeout 1000 ms). Trace:
W20250628 07:52:46.913163   541 rpcz_store.cc:269] 0628 07:52:45.851527 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:46.912540 (+1061013us) service_pool.cc:212] Skipping call since client already timed out
0628 07:52:46.912760 (+   220us) inbound_call.cc:173] Queueing failure response
Metrics: {}
W20250628 07:52:46.915050   544 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
W20250628 07:52:46.915690   543 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.920594  2765 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:46.931463  2769 raft_consensus.cc:513] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:46.939380  2142 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276"
I20250628 07:52:46.939888  2142 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 4 FOLLOWER]: Advancing to term 5
I20250628 07:52:46.908951  1230 raft_consensus.cc:383] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.941243  1230 raft_consensus.cc:738] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
W20250628 07:52:46.939906   547 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.941932  1230 consensus_queue.cc:260] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:46.945547   545 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.946722  2142 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 023ed4cb582a44509bccf42b52c494d7 in term 5.
I20250628 07:52:46.947010  1230 ts_tablet_manager.cc:1428] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.041s	user 0.005s	sys 0.000s
W20250628 07:52:46.947826   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.947922   531 leader_election.cc:304] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276; no voters: 
W20250628 07:52:46.950474   544 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.950579  1229 raft_consensus.cc:357] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.951226  1229 raft_consensus.cc:383] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:46.951483  1229 raft_consensus.cc:738] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:46.952100  1229 consensus_queue.cc:260] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:46.953153   543 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.960332  2658 raft_consensus.cc:491] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:46.960844  2658 raft_consensus.cc:513] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:46.962643  2658 leader_election.cc:290] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:46.964218   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d3501090b0a34ca9882dde55b19e3621" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:46.965332  1062 leader_election.cc:343] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:46.971632   544 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.993858  1229 ts_tablet_manager.cc:1428] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.081s	user 0.005s	sys 0.000s
I20250628 07:52:46.947921  1230 tablet_bootstrap.cc:492] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:46.976888   542 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.977324  2765 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 5 FOLLOWER]: Leader election won for term 5
W20250628 07:52:46.983407   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
I20250628 07:52:46.990758  2769 leader_election.cc:290] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [CANDIDATE]: Term 5 election: Requested vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 992ac4c27fb744deb1edf7765553ef08 (127.0.107.188:42677)
I20250628 07:52:47.013595  1108 tablet_service.cc:1515] Processing DeleteTablet for tablet 7e2ecd14e8524f018dd3c0aecd564218 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:52:42 UTC) from {username='slave'} at 127.0.0.1:54948
I20250628 07:52:47.013860  2765 raft_consensus.cc:695] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 5 LEADER]: Becoming Leader. State: Replica: 023ed4cb582a44509bccf42b52c494d7, State: Running, Role: LEADER
W20250628 07:52:47.013927   533 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:47.014724  2765 consensus_queue.cc:237] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 555, Committed index: 555, Last appended: 3.555, Last appended by leader: 555, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
W20250628 07:52:47.019116   546 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
I20250628 07:52:47.020442  2658 raft_consensus.cc:491] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.020885  1229 tablet_bootstrap.cc:492] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:47.020862  2658 raft_consensus.cc:513] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.022861  2658 leader_election.cc:290] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:47.025153   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "77234b63748849b6996c7aa0ca23ef05" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:47.026350  1062 leader_election.cc:343] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:52:47.020249  1230 tablet_bootstrap.cc:654] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
W20250628 07:52:47.020938   544 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
W20250628 07:52:47.032763   543 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
W20250628 07:52:47.003463   540 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER
W20250628 07:52:47.023478   545 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
W20250628 07:52:47.038846   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
I20250628 07:52:47.040264  2776 tablet_replica.cc:331] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: stopping tablet replica
I20250628 07:52:47.042969  2776 raft_consensus.cc:2241] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Raft consensus shutting down.
W20250628 07:52:47.043838   545 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
W20250628 07:52:47.049485   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
I20250628 07:52:47.050833  2776 raft_consensus.cc:2270] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806 [term 2 FOLLOWER]: Raft consensus is shut down!
W20250628 07:52:47.051947   539 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
I20250628 07:52:47.042157  1230 tablet_bootstrap.cc:492] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.053189  1230 ts_tablet_manager.cc:1397] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.105s	user 0.009s	sys 0.001s
I20250628 07:52:47.055634  1230 raft_consensus.cc:357] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.061141  2776 ts_tablet_manager.cc:1905] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:52:47.061002  2657 consensus_queue.cc:1035] T f9a553c69c254fdcaf413b53018bbfb2 P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:47.056231  1230 raft_consensus.cc:383] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.065778  1230 raft_consensus.cc:738] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.066385  1230 consensus_queue.cc:260] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.072494  1230 ts_tablet_manager.cc:1428] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.019s	user 0.005s	sys 0.000s
I20250628 07:52:47.077697  1230 tablet_bootstrap.cc:492] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.078547  1229 tablet_bootstrap.cc:654] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.059307  2769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 5 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:47.080001  2769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
W20250628 07:52:47.061547   542 catalog_manager.cc:6120] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: An error occurred while updating sys-tables: Service unavailable: leader is not yet ready
W20250628 07:52:47.027364   613 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.189:33023, user_credentials={real_user=slave}} blocked reactor thread for 246517us
I20250628 07:52:47.090397   555 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [term 5 LEADER]: Rejecting Update request from peer 992ac4c27fb744deb1edf7765553ef08 for earlier term 4. Current term is 5. Ops: [4.556-4.556]
I20250628 07:52:47.098105  1230 tablet_bootstrap.cc:654] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.115936  2762 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 }, Status: INVALID_TERM, Last received: 3.555, Next index: 556, Last known committed idx: 555, Time since last communication: 0.000s
I20250628 07:52:47.116617  2762 consensus_queue.cc:1230] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [LEADER]: Peer 023ed4cb582a44509bccf42b52c494d7 log is divergent from this leader: its last log entry 5.556 is not in this leader's log and it has not received anything from this leader yet. Falling back to committed index 555
I20250628 07:52:47.117648  2761 raft_consensus.cc:3053] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 LEADER]: Stepping down as leader of term 4
I20250628 07:52:47.117935  2761 raft_consensus.cc:738] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 LEADER]: Becoming Follower/Learner. State: Replica: 992ac4c27fb744deb1edf7765553ef08, State: Running, Role: LEADER
I20250628 07:52:47.118618  2761 consensus_queue.cc:260] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 555, Committed index: 555, Last appended: 4.556, Last appended by leader: 556, Current term: 4, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:47.120298  2768 ts_tablet_manager.cc:1918] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 2.1
I20250628 07:52:47.120687  2768 log.cc:1199] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/7e2ecd14e8524f018dd3c0aecd564218
I20250628 07:52:47.121454  2768 ts_tablet_manager.cc:1939] T 7e2ecd14e8524f018dd3c0aecd564218 P 82f43cdfab544b17a6a71033983e65d4: Deleting consensus metadata
W20250628 07:52:47.132776   539 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 70) took 4024 ms (client timeout 1000 ms). Trace:
I20250628 07:52:47.135177  2776 ts_tablet_manager.cc:1918] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 2.1
I20250628 07:52:47.140156   532 catalog_manager.cc:4928] TS 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): tablet 7e2ecd14e8524f018dd3c0aecd564218 (table default.Table_0979356e8b1a441fa942c0359545cfc9 [id=e518ccb92cf248d09e0f974067eab1bd]) successfully deleted
I20250628 07:52:47.141336  2776 log.cc:1199] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/7e2ecd14e8524f018dd3c0aecd564218
I20250628 07:52:47.142242  2776 ts_tablet_manager.cc:1939] T 7e2ecd14e8524f018dd3c0aecd564218 P 6a58514e01ea4989809bc2992fb2b806: Deleting consensus metadata
I20250628 07:52:47.141978  2740 tablet_loader.cc:96] loaded metadata for tablet 70584cf82896499ab43f7282337078ef (table default.Table_dddf2413c21f4d4e8c3da5c33cef8a01 [id=839e2998d8e642f9a4fd0f7d376cefa4])
I20250628 07:52:47.146688  1230 tablet_bootstrap.cc:492] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.147132  1230 ts_tablet_manager.cc:1397] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.070s	user 0.011s	sys 0.000s
I20250628 07:52:47.158116  2761 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 4 FOLLOWER]: Advancing to term 5
W20250628 07:52:47.165376   613 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.189:33023, user_credentials={real_user=slave}} blocked reactor thread for 51710.1us
I20250628 07:52:47.174108  1230 raft_consensus.cc:357] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.174758  1230 raft_consensus.cc:383] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.175020  1230 raft_consensus.cc:738] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.175588  1230 consensus_queue.cc:260] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.178570  2658 raft_consensus.cc:491] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.178997  2658 raft_consensus.cc:513] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.180846  2658 leader_election.cc:290] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:47.182902   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c3239c3041c84dfdbc61de4fcb4ea5dd" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
W20250628 07:52:47.183902  1062 leader_election.cc:343] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061): Illegal state: must be running to vote when last-logged opid is not known
W20250628 07:52:47.133987   539 rpcz_store.cc:269] 0628 07:52:43.108681 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.113040 (+  4359us) service_pool.cc:225] Handling call
0628 07:52:43.145391 (+ 32351us) spinlock_profiling.cc:243] Waited 25.7 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:46.902323 (+3756932us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.902344 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:46.904948 (+  2604us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.904976 (+    28us) write_op.cc:454] Released schema lock
0628 07:52:46.947632 (+ 42656us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.947654 (+    22us) write_op.cc:454] Released schema lock
0628 07:52:46.983211 (+ 35557us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.983231 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:47.038648 (+ 55417us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.038667 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:47.049310 (+ 10643us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.049329 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:47.051773 (+  2444us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.051792 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:47.132511 (+ 80719us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:44.265026 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:44.265414 (+   388us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:44.265438 (+    24us) write_op.cc:435] Acquired schema lock
0628 07:52:44.265450 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:44.265836 (+   386us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:44.265873 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:44.265895 (+    22us) tablet.cc:683] Acquiring locks for 3 operations
0628 07:52:44.266116 (+   221us) tablet.cc:699] Row locks acquired
0628 07:52:44.266128 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:44.266231 (+   103us) write_op.cc:270] Start()
0628 07:52:44.266305 (+    74us) write_op.cc:276] Timestamp: P: 1751097164266156 usec, L: 0
0628 07:52:44.266320 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:44.266763 (+   443us) log.cc:844] Serialized 1293 byte log entry
0628 07:52:44.275931 (+  9168us) op_driver.cc:464] REPLICATION: finished
0628 07:52:44.276234 (+   303us) write_op.cc:301] APPLY: starting
0628 07:52:44.276282 (+    48us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:44.276397 (+   115us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:44.276487 (+    90us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:44.276841 (+   354us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:44.276979 (+   138us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=3
0628 07:52:44.277022 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:44.277558 (+   536us) log.cc:844] Serialized 35 byte log entry
0628 07:52:44.277696 (+   138us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:44.277769 (+    73us) write_op.cc:454] Released schema lock
0628 07:52:44.277799 (+    30us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:44.278012 (+   213us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:44.278033 (+    21us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:46.902153 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.902185 (+    32us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.902207 (+    22us) write_op.cc:454] Released schema lock
0628 07:52:46.902224 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.904770 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.904799 (+    29us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.904818 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.904835 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.947438 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.947475 (+    37us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.947493 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.947514 (+    21us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.983044 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.983072 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.983090 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.983107 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.038447 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.038472 (+    25us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.038490 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:47.038506 (+    16us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.049162 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.049190 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.049207 (+    17us) write_op.cc:454] Released schema lock
0628 07:52:47.049224 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.051591 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.051635 (+    44us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.051663 (+    28us) write_op.cc:454] Released schema lock
0628 07:52:47.051680 (+    17us) write_op.cc:335] FINISH: Op aborted
Metrics: {"mutex_wait_us":48855,"reported_tablets":41,"spinlock_wait_cycles":59201792,"child_traces":[["op",{"apply.queue_time_us":175,"apply.run_cpu_time_us":1441,"apply.run_wall_time_us":1751,"num_ops":3,"prepare.queue_time_us":92,"prepare.run_cpu_time_us":1983,"prepare.run_wall_time_us":1982,"raft.queue_time_us":1130,"raft.run_cpu_time_us":598,"raft.run_wall_time_us":595,"replication_time_us":9518}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{}]]}
I20250628 07:52:47.196452  1230 ts_tablet_manager.cc:1428] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.049s	user 0.005s	sys 0.006s
I20250628 07:52:47.197469  1230 tablet_bootstrap.cc:492] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
W20250628 07:52:47.200500  2780 log.cc:927] Time spent T f9a553c69c254fdcaf413b53018bbfb2 P 6a58514e01ea4989809bc2992fb2b806: Append to log took a long time: real 0.075s	user 0.000s	sys 0.000s
I20250628 07:52:47.203194  1229 tablet_bootstrap.cc:492] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:47.203573  1229 ts_tablet_manager.cc:1397] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.183s	user 0.009s	sys 0.000s
I20250628 07:52:47.222555   533 catalog_manager.cc:4928] TS 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709): tablet 7e2ecd14e8524f018dd3c0aecd564218 (table default.Table_0979356e8b1a441fa942c0359545cfc9 [id=e518ccb92cf248d09e0f974067eab1bd]) successfully deleted
I20250628 07:52:47.230549  2740 tablet_loader.cc:96] loaded metadata for tablet 71c5bdface3d44ecae33bef018e7f18d (table default.Table_6fd35d7894eb4e9891f539a62d743e62 [id=700083322b4a4b118855057493210fb7])
I20250628 07:52:47.231590  1229 raft_consensus.cc:357] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.237778  2740 tablet_loader.cc:96] loaded metadata for tablet 71c83c726809488ea0be8f724189e8e2 (table default.Table_123910cc7356492f93d0e1d1df124ad9 [id=5a391fa6a42f4b828a6eb4cc5e76465c])
I20250628 07:52:47.238214  1229 raft_consensus.cc:383] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.238291  1230 tablet_bootstrap.cc:654] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.238610  1229 raft_consensus.cc:738] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
W20250628 07:52:47.240540   534 catalog_manager.cc:4726] Async tablet task AlterTable RPC for tablet 71c5bdface3d44ecae33bef018e7f18d (table default.Table_6fd35d7894eb4e9891f539a62d743e62 [id=700083322b4a4b118855057493210fb7], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 71c5bdface3d44ecae33bef018e7f18d consensus state has no leader
I20250628 07:52:47.259938  2139 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 5 FOLLOWER]: Rejecting Update request from peer 992ac4c27fb744deb1edf7765553ef08 for earlier term 4. Current term is 5. Ops: [3.418-4.556]
I20250628 07:52:47.262156  1230 tablet_bootstrap.cc:492] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.262512  1230 ts_tablet_manager.cc:1397] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.065s	user 0.010s	sys 0.000s
I20250628 07:52:47.264669  1230 raft_consensus.cc:357] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.265286  1230 raft_consensus.cc:383] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.265534  1230 raft_consensus.cc:738] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.266278  1230 consensus_queue.cc:260] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.271891  1230 ts_tablet_manager.cc:1428] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.009s	user 0.006s	sys 0.000s
I20250628 07:52:47.296677  1230 tablet_bootstrap.cc:492] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.303810  1230 tablet_bootstrap.cc:654] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.307260  1229 consensus_queue.cc:260] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.310421  1229 ts_tablet_manager.cc:1428] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4: Time spent starting tablet: real 0.107s	user 0.006s	sys 0.000s
I20250628 07:52:47.244166  2740 tablet_loader.cc:96] loaded metadata for tablet 71d772c683f94e79bd14bfc139357844 (table default.Table_21bfb5f55d7a4676a0054415d0b5acb6 [id=51bf279dfab8478eab92be5b0685e013])
I20250628 07:52:47.323521  2740 tablet_loader.cc:96] loaded metadata for tablet 71da5208ceef4186855cbe471f0fd8a3 (table default.Table_177b8783c82149aabfddcfe195d451a7 [id=3f993fcff5574851b92244167bb2ee8e])
I20250628 07:52:47.326273  2740 tablet_loader.cc:96] loaded metadata for tablet 7986ee140247478eafc2d95aaec8f8b4 (table default.Table_0b57fdd311634de995ecfba2548d0468 [id=e5faef7f22b84218aa7d378241a65d43])
I20250628 07:52:47.327386  2740 tablet_loader.cc:96] loaded metadata for tablet 79d15e600b8b460fab54eb21193f1f40 (table default.Table_06835b5c509a43aca747e21ac8381c29 [id=38af28a507664aa69450e8bdc5b0533a])
I20250628 07:52:47.333247  2658 raft_consensus.cc:491] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.333639  2658 raft_consensus.cc:513] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.335363  2658 leader_election.cc:290] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:47.336949   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af28ae2139b2417381808cb4068ddf60" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4" is_pre_election: true
I20250628 07:52:47.337441   984 raft_consensus.cc:2466] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 0.
I20250628 07:52:47.338593  1062 leader_election.cc:304] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:47.339344  2658 raft_consensus.cc:2802] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:47.339636  2658 raft_consensus.cc:491] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:47.339951  1229 tablet_bootstrap.cc:492] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4: Bootstrap starting.
I20250628 07:52:47.339932  2658 raft_consensus.cc:3058] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:47.342083  2699 consensus_queue.cc:1035] T d9e0ea5e300347c7adc8e30b9070976b P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:52:47.347200  2658 raft_consensus.cc:513] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.358789  2699 raft_consensus.cc:491] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.359211  2699 raft_consensus.cc:513] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.360231  1229 tablet_bootstrap.cc:654] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.364667  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "296d777dcbfb4d5aa80db78e3f07426c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806" is_pre_election: true
I20250628 07:52:47.365234  1121 raft_consensus.cc:2466] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 0.
I20250628 07:52:47.366255   921 leader_election.cc:304] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:47.371811  2715 raft_consensus.cc:2802] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:52:47.372162  2715 raft_consensus.cc:491] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:52:47.372504  2715 raft_consensus.cc:3058] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:47.378341  1230 tablet_bootstrap.cc:492] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.378827  1230 ts_tablet_manager.cc:1397] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.082s	user 0.008s	sys 0.003s
I20250628 07:52:47.379063  2715 raft_consensus.cc:513] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.381377  2715 leader_election.cc:290] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Requested vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:47.383101  1121 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "296d777dcbfb4d5aa80db78e3f07426c" candidate_uuid: "82f43cdfab544b17a6a71033983e65d4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a58514e01ea4989809bc2992fb2b806"
I20250628 07:52:47.383538  1229 tablet_bootstrap.cc:492] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4: No bootstrap required, opened a new log
I20250628 07:52:47.383649  1121 raft_consensus.cc:3058] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:47.387161   984 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "af28ae2139b2417381808cb4068ddf60" candidate_uuid: "6a58514e01ea4989809bc2992fb2b806" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82f43cdfab544b17a6a71033983e65d4"
I20250628 07:52:47.387657   984 raft_consensus.cc:3058] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:52:47.390043  1229 ts_tablet_manager.cc:1397] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4: Time spent bootstrapping tablet: real 0.050s	user 0.010s	sys 0.000s
I20250628 07:52:47.391515  1121 raft_consensus.cc:2466] T 296d777dcbfb4d5aa80db78e3f07426c P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 82f43cdfab544b17a6a71033983e65d4 in term 1.
I20250628 07:52:47.392585   921 leader_election.cc:304] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:47.393215  2657 raft_consensus.cc:2802] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:47.393788  2657 raft_consensus.cc:695] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [term 1 LEADER]: Becoming Leader. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Running, Role: LEADER
W20250628 07:52:47.394284   532 connection.cc:537] client connection to 127.0.107.129:34311 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.394354  1062 connection.cc:537] client connection to 127.0.107.129:34311 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.394757  1062 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.394721   532 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 2 similar messages]
I20250628 07:52:47.394483  2657 consensus_queue.cc:237] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.369673  2699 leader_election.cc:290] T 296d777dcbfb4d5aa80db78e3f07426c P 82f43cdfab544b17a6a71033983e65d4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311), 6a58514e01ea4989809bc2992fb2b806 (127.0.107.131:37709)
I20250628 07:52:47.397856   984 raft_consensus.cc:2466] T af28ae2139b2417381808cb4068ddf60 P 82f43cdfab544b17a6a71033983e65d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a58514e01ea4989809bc2992fb2b806 in term 1.
I20250628 07:52:47.392549  1229 raft_consensus.cc:357] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.401275  1229 raft_consensus.cc:383] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.401587  1229 raft_consensus.cc:738] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82f43cdfab544b17a6a71033983e65d4, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.402700  2658 leader_election.cc:290] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Requested vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:47.403577   533 connection.cc:537] server connection from 127.0.107.129:43357 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.404086   920 connection.cc:537] client connection to 127.0.107.129:34311 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.404413   920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20250628 07:52:47.405320  1229 consensus_queue.cc:260] T 01b1767597424753a588090e9228d133 P 82f43cdfab544b17a6a71033983e65d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.408757   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 877
I20250628 07:52:47.411854  2787 raft_consensus.cc:491] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.412361  2787 raft_consensus.cc:513] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.414598  2787 leader_election.cc:290] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:47.415254   613 connection.cc:537] server connection from 127.0.107.129:52405 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.424430   612 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:47.381163  1230 raft_consensus.cc:357] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.425486  1230 raft_consensus.cc:383] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.425746  1230 raft_consensus.cc:738] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.435889  2142 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5fd67221fa5f424ea0f83e582629d276 [term 5 FOLLOWER]: Refusing update from remote peer 023ed4cb582a44509bccf42b52c494d7: Log matching property violated. Preceding OpId in replica: term: 3 index: 417. Preceding OpId from leader: term: 5 index: 557. (index mismatch)
I20250628 07:52:47.440132  2769 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 556, Last known committed idx: 417, Time since last communication: 0.000s
I20250628 07:52:47.445845  1230 consensus_queue.cc:260] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.447837  1230 ts_tablet_manager.cc:1428] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.069s	user 0.003s	sys 0.000s
I20250628 07:52:47.448632  1230 tablet_bootstrap.cc:492] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.458048  1062 leader_election.cc:304] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806, 82f43cdfab544b17a6a71033983e65d4; no voters: 
I20250628 07:52:47.458761  2787 raft_consensus.cc:2802] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:52:47.459154  2787 raft_consensus.cc:695] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [term 1 LEADER]: Becoming Leader. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Running, Role: LEADER
W20250628 07:52:47.459297  1062 leader_election.cc:336] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.459662  1062 leader_election.cc:304] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:47.459894  2787 consensus_queue.cc:237] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
W20250628 07:52:47.460417  1062 leader_election.cc:336] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.460755  1062 leader_election.cc:304] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
I20250628 07:52:47.461540  2786 raft_consensus.cc:2747] T d3501090b0a34ca9882dde55b19e3621 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:47.469614  1062 leader_election.cc:336] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.470156  1062 leader_election.cc:304] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a58514e01ea4989809bc2992fb2b806; no voters: 6005c44dfe4848f18a304621cd49e934, 82f43cdfab544b17a6a71033983e65d4
W20250628 07:52:47.470173   532 catalog_manager.cc:4671] TS 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 7e2ecd14e8524f018dd3c0aecd564218: Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.470795  2786 raft_consensus.cc:2747] T 77234b63748849b6996c7aa0ca23ef05 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:52:47.471268  2786 raft_consensus.cc:2747] T c3239c3041c84dfdbc61de4fcb4ea5dd P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:52:47.472149  1062 consensus_peers.cc:489] T 181c57eeb50745beb09c36a38b75d9be P 6a58514e01ea4989809bc2992fb2b806 -> Peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Couldn't send request to peer 6005c44dfe4848f18a304621cd49e934. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20250628 07:52:47.472532  1062 leader_election.cc:336] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.483708  2740 tablet_loader.cc:96] loaded metadata for tablet 7ae51bcf0bd94770a818b504749184fc (table default.Table_061d0f52cd51486a88f1e59480dffc7f [id=da004477b85348e69492c13b6404eed4])
W20250628 07:52:47.504762  1062 leader_election.cc:336] T af28ae2139b2417381808cb4068ddf60 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
W20250628 07:52:47.505424  1062 leader_election.cc:336] T abf8f2d8069445bab27852e4e16a6bee P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.519016  1230 tablet_bootstrap.cc:654] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.515566  2740 tablet_loader.cc:96] loaded metadata for tablet 7d2a96449ed44d76a015e929d6a6edb0 (table default.Table_5baf5e16d9f6482ab94b15a2706e3fbd [id=77ef1587831e44bd9a93a373b6fd9d0f])
I20250628 07:52:47.523568  2787 raft_consensus.cc:491] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.523993  2787 raft_consensus.cc:513] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.527913  2787 leader_election.cc:290] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
W20250628 07:52:47.532157  1062 leader_election.cc:336] T f0c4343648e144c9922d9e6c484abb71 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.564752  2740 tablet_loader.cc:96] loaded metadata for tablet 7e3210014d264071bfc2960a7406bcd7 (table default.Table_528bf935d76946ddb897d5bf719bca19 [id=3a6dc82e4c5846aebf2fe22d8c31bd83])
I20250628 07:52:47.591212  1230 tablet_bootstrap.cc:492] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.591760  1230 ts_tablet_manager.cc:1397] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.143s	user 0.010s	sys 0.000s
W20250628 07:52:47.593200   533 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
I20250628 07:52:47.588389  2740 tablet_loader.cc:96] loaded metadata for tablet 7eca25db66fa42a798fc002d43f7e3c3 (table default.Table_62804f58107647eb82169b3550598170 [id=be8d8530baf34f4ea04d59598d765502])
I20250628 07:52:47.595023  1230 raft_consensus.cc:357] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.595894  1230 raft_consensus.cc:383] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.596217  1230 raft_consensus.cc:738] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.598001  1230 consensus_queue.cc:260] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.614140  2740 tablet_loader.cc:96] loaded metadata for tablet 80ee935a2d58422fb1bfbc9103940920 (table default.Table_22a7acccf75140a8a636cec2a73177f6 [id=a9599116e3c04051a9484736a07d350f])
I20250628 07:52:47.628710  1230 ts_tablet_manager.cc:1428] T 7986ee140247478eafc2d95aaec8f8b4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.037s	user 0.005s	sys 0.000s
I20250628 07:52:47.633427  1230 tablet_bootstrap.cc:492] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.647343  1230 tablet_bootstrap.cc:654] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.669054  2740 tablet_loader.cc:96] loaded metadata for tablet 81d9066872654fceb44b7fb63b60f070 (table default.Table_998846661d2441bfba53057917d7f1bf [id=48ef616b338f4d7b932257bfb1b9f0be])
I20250628 07:52:47.687278  2740 tablet_loader.cc:96] loaded metadata for tablet 8226c926201040c993804d2ae8450202 (table default.Table_bda937abf44b485aa9fd46ab195442f8 [id=2a211e6249e24fc89a60604f187a5554])
I20250628 07:52:47.696223  1230 tablet_bootstrap.cc:492] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.696681  1230 ts_tablet_manager.cc:1397] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.063s	user 0.009s	sys 0.000s
I20250628 07:52:47.699043  1230 raft_consensus.cc:357] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.699663  1230 raft_consensus.cc:383] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.699904  1230 raft_consensus.cc:738] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.700650  1230 consensus_queue.cc:260] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.702834  1230 ts_tablet_manager.cc:1428] T f026cebba0814da3acbc26669d0211d4 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20250628 07:52:47.705674  2740 tablet_loader.cc:96] loaded metadata for tablet 83766eca91da416797165dbb0262f9cb (table default.Table_d3053995fff94245ae716d9ea4118068 [id=e1a9ab2b162b43399279afd19dd0f093])
I20250628 07:52:47.708962  2740 tablet_loader.cc:96] loaded metadata for tablet 85b3d2d151b549d6967e0da2a57723f0 (table default.Table_db0904127a6144d7acb537f17d6891d6 [id=f8b5a176637544b0b21ec14c5bb5598f])
I20250628 07:52:47.728565  1230 tablet_bootstrap.cc:492] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.734566  2787 raft_consensus.cc:491] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.735112  2787 raft_consensus.cc:513] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.735869  2740 tablet_loader.cc:96] loaded metadata for tablet 85c0ba070a8a40f2b32854ca24bc025c (table default.Table_04dc0144b3954979b9fdcce449c44782 [id=ad5a5ac2a4024b439c1d4fbe5e82509d])
I20250628 07:52:47.736976  2740 tablet_loader.cc:96] loaded metadata for tablet 86591ef37c8547189ff8ef270e9bfa9a (table default.Table_bc0991297d684c8fb6295c5ed29cd3c1 [id=0bcb503dea9b4b9b96ae9a3f6bb9f9a5])
I20250628 07:52:47.738049  2740 tablet_loader.cc:96] loaded metadata for tablet 872ebf0393fa40cf82feebaad1ca3fa3 (table default.Table_f84ba40fc1d14d55a6725054f1c3465f [id=5e0bf38f7dc94bf59a2f39456ecae8fc])
I20250628 07:52:47.739061  2740 tablet_loader.cc:96] loaded metadata for tablet 878b25ab3282423fab41014a8ea8949c (table default.Table_d9f1fae1f46f45efbde92d35775fdd0e [id=e32e5fdf4ddb42d794ebe9d70ad300fb])
I20250628 07:52:47.740782  2740 tablet_loader.cc:96] loaded metadata for tablet 8a0f311c619443198e33dfef20a3f4a4 (table default.Table_ade87cf74e6b498690118e3541e41383 [id=2ccbb0163b564f39971348853bdbd012])
W20250628 07:52:47.741925  1062 leader_election.cc:336] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.742086  1230 tablet_bootstrap.cc:654] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.742512  2787 leader_election.cc:290] T 2595308cbc0d44f0abfd0a126f16d4df P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:47.755370  2808 raft_consensus.cc:491] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.760035  2808 raft_consensus.cc:513] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } }
I20250628 07:52:47.762027  2740 tablet_loader.cc:96] loaded metadata for tablet 8bcc01e64a874cd69300e3b78e2d9ab8 (table default.Table_c22cd1bf6d884c33bf39513fb5da717a [id=2f1a522e39a24e749aaaa0578ad3c6c2])
I20250628 07:52:47.762868   555 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 6 candidate_status { last_received { term: 4 index: 556 } } ignore_live_leader: false dest_uuid: "023ed4cb582a44509bccf42b52c494d7" is_pre_election: true
I20250628 07:52:47.763207  2740 tablet_loader.cc:96] loaded metadata for tablet 8c1491faa1be485792b6193d4ca22f37 (table default.Table_7924c3ed1cf34beba7ba362d781f34de [id=ca323f84b00048e69461d15b7bd6369a])
I20250628 07:52:47.762954  2139 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "992ac4c27fb744deb1edf7765553ef08" candidate_term: 6 candidate_status { last_received { term: 4 index: 556 } } ignore_live_leader: false dest_uuid: "5fd67221fa5f424ea0f83e582629d276" is_pre_election: true
I20250628 07:52:47.764241  2740 tablet_loader.cc:96] loaded metadata for tablet 8d86b6da48254a608346474cf33be345 (table default.Table_df7fc4695669432f9b2bcc300360c70b [id=065317d32ef7414aa62609e873462803])
I20250628 07:52:47.766268   610 leader_election.cc:304] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 992ac4c27fb744deb1edf7765553ef08; no voters: 023ed4cb582a44509bccf42b52c494d7, 5fd67221fa5f424ea0f83e582629d276
I20250628 07:52:47.769184  2808 leader_election.cc:290] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 5fd67221fa5f424ea0f83e582629d276 (127.0.107.190:37209), 023ed4cb582a44509bccf42b52c494d7 (127.0.107.189:33023)
I20250628 07:52:47.770589  1230 tablet_bootstrap.cc:492] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.771059  2808 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
I20250628 07:52:47.778278  2740 tablet_loader.cc:96] loaded metadata for tablet 8e843597f12144799e7e13a7a4b1cc56 (table default.Table_3f9fa575c08344b9965c9fb9bccc8c85 [id=a8e8d7b379564c7699f75b69b507bb73])
I20250628 07:52:47.781738  1230 ts_tablet_manager.cc:1397] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.053s	user 0.008s	sys 0.002s
I20250628 07:52:47.783798  2740 tablet_loader.cc:96] loaded metadata for tablet 8feee3be1a664cae9da417b190163a94 (table default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89])
I20250628 07:52:47.770656  2787 raft_consensus.cc:491] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:52:47.784452  2787 raft_consensus.cc:513] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.785013  2740 tablet_loader.cc:96] loaded metadata for tablet 9045c6953fb64629a2aeb59b498f6d9a (table default.Table_7631c6a0720642949d4b63e271a92e4c [id=46657c237cc041279dbb9e365d574c01])
I20250628 07:52:47.784894  1230 raft_consensus.cc:357] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.785728  1230 raft_consensus.cc:383] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.786005  1230 raft_consensus.cc:738] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.786293  2787 leader_election.cc:290] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82f43cdfab544b17a6a71033983e65d4 (127.0.107.130:35061), 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311)
I20250628 07:52:47.786876  2740 tablet_loader.cc:96] loaded metadata for tablet 9130fcc0c37f4d8e8852719fbafc3695 (table default.Table_fe7551bb42dd452ab32b051075e3398b [id=6ad3b42bbcbd4d5e894480654bf6bebe])
I20250628 07:52:47.787197  1230 consensus_queue.cc:260] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.787984  2740 tablet_loader.cc:96] loaded metadata for tablet 924a153449b64d41b7db8c46afed93a9 (table default.Table_b0dd0b3a3f1e4ba9b36cd25c539c0a00 [id=afffd13a151540f6863fc55f1a0c3f8c])
I20250628 07:52:47.789037  2740 tablet_loader.cc:96] loaded metadata for tablet 924b921830b7403596ebcc6cf80dc24d (table default.Table_c62a447971794f91bd3b8d3b7957badf [id=4d89287c4bac4bceaa6aaab65010f94a])
I20250628 07:52:47.790134  2740 tablet_loader.cc:96] loaded metadata for tablet 928ffc3fd77d4ffe9c59c96afb436838 (table default.Table_f6dc476a6f3b42afa13683cc9611a247 [id=6514b4f905774cf3bfec0ddc73804534])
W20250628 07:52:47.792094  1062 leader_election.cc:336] T 8d86b6da48254a608346474cf33be345 P 6a58514e01ea4989809bc2992fb2b806 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6005c44dfe4848f18a304621cd49e934 (127.0.107.129:34311): Network error: Client connection negotiation failed: client connection to 127.0.107.129:34311: connect: Connection refused (error 111)
I20250628 07:52:47.800112  2740 tablet_loader.cc:96] loaded metadata for tablet 95fa7c0a68d34d728042fb5a8a3b7307 (table default.Table_07319c0694ca4307b243e8938d21939b [id=11039cb768f5492c88f0970b4aae370e])
I20250628 07:52:47.801441  2740 tablet_loader.cc:96] loaded metadata for tablet 987bf94636bd48b3a428ff7cc3b6bedc (table default.Table_753078c239dc43a6a5125732a2ff887c [id=eeec5abe55af475f8195a9ec1880ca62])
I20250628 07:52:47.801514  1230 ts_tablet_manager.cc:1428] T 670f91ee94f7460caa0790e232ac7fa8 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.019s	user 0.004s	sys 0.001s
I20250628 07:52:47.802316  1230 tablet_bootstrap.cc:492] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.803815  2740 tablet_loader.cc:96] loaded metadata for tablet 9ba579ef503c427295b2ba6185aa3ccb (table default.Table_c97313f05be54e4da2bc4cbbc04c8682 [id=b98b8985d5ad44b1a1618aa721533c5c])
I20250628 07:52:47.811997  2740 tablet_loader.cc:96] loaded metadata for tablet a168c27bfea94f9bba06c3a88329a892 (table default.Table_c974994f851542aa92291ccc82d531e0 [id=f7cf0e3d20f645d2ad5d9b5ee3637834])
I20250628 07:52:47.813637  2740 tablet_loader.cc:96] loaded metadata for tablet a353cc9eedf34cc7aa4cfbcc7198247b (table default.Table_09dfad9563454a0e9624d7e15961a374 [id=dce9747fa94f4f86bdafae93dbf53339])
I20250628 07:52:47.815347  2740 tablet_loader.cc:96] loaded metadata for tablet a4900b15c92a4024917480a3fce8373c (table default.Table_4f9faad6d70b4a559cfacf4e5deb27e3 [id=54d01002870a488baf5a808adbfbc006])
I20250628 07:52:47.816390  2740 tablet_loader.cc:96] loaded metadata for tablet a5ac50e627c0470782f96bd77a61f763 (table default.Table_5ad5f13abefc445b8c69ab0be547644d [id=033b1fae1f634663a3efa53e00c858ac])
I20250628 07:52:47.818102  2740 tablet_loader.cc:96] loaded metadata for tablet abf8f2d8069445bab27852e4e16a6bee (table default.Table_e10aee64839f49a18fc40406bb808709 [id=d607ae35fec24cab9d45e70c9ec351f3])
I20250628 07:52:47.819837  2740 tablet_loader.cc:96] loaded metadata for tablet ad4a43b1e1074531ba41ed08827ffa38 (table default.Table_4107fb91f5704f38b337a443ee54704c [id=d4a373f636634fd3ac4b34d1f31ca380])
I20250628 07:52:47.831693  1230 tablet_bootstrap.cc:654] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.820951  2740 tablet_loader.cc:96] loaded metadata for tablet adb7506b712243fe91bccc326d39cdd7 (table default.Table_481f2aef57d748fa98e48e26b106b1f5 [id=0335cd54e0504a65ad54d973365cd75b])
I20250628 07:52:47.838266  2740 tablet_loader.cc:96] loaded metadata for tablet aea9552ebd5949c8b0b5f23e132e9b6c (table default.Table_6c95fbef63ae4a34aff4518c2d8cbdad [id=4accce061ee5447187ab61870ffc1e77])
I20250628 07:52:47.839395  2740 tablet_loader.cc:96] loaded metadata for tablet aec37016d1674945b399cfc2f92b08fc (table default.Table_bb80571048c94b099e096f7c712af383 [id=4a58d7983dd146ae9e4ae45df899b0b6])
I20250628 07:52:47.843652  2740 tablet_loader.cc:96] loaded metadata for tablet b1da78a71a634d408fb5de4efe92ebbe (table default.Table_324d60f6b2914fdb85b95e4201ffee43 [id=0b6ea0c021b3450ba01083f4373e29e8])
I20250628 07:52:47.844792  2740 tablet_loader.cc:96] loaded metadata for tablet b1fcc569fb1b4a84983b651074e3e64d (table default.Table_3c5a83219fbc4befa57b1095ee0e4fa0 [id=69980641f09443da87a0d85cb5cca3aa])
I20250628 07:52:47.845959  2740 tablet_loader.cc:96] loaded metadata for tablet b2bd3fccfe524ceaa7ff1a566ca72631 (table default.Table_8ddcbb3e78904ccf91355f6b6d7ef9f0 [id=3dfaaca8b9384537917d5e8262b416d3])
I20250628 07:52:47.860613  1230 tablet_bootstrap.cc:492] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.861035  1230 ts_tablet_manager.cc:1397] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.059s	user 0.001s	sys 0.008s
I20250628 07:52:47.863253  2740 tablet_loader.cc:96] loaded metadata for tablet b69bbcd5fede44ed9c4e1a2c35338d56 (table default.Table_5c036fde993047db95ee90d9ffd4be8d [id=640ae94271994216b7cbf2ffdef1cec9])
I20250628 07:52:47.863713  1230 raft_consensus.cc:357] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.864326  1230 raft_consensus.cc:383] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.864459  2740 tablet_loader.cc:96] loaded metadata for tablet b759dcf28b784ab99b7aed5d22c651dc (table default.Table_9bd0a6cbc9e8466cb54f871dc9938e95 [id=1234c3ae728a4b9fb829fed377c84833])
I20250628 07:52:47.864573  1230 raft_consensus.cc:738] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.865664  2740 tablet_loader.cc:96] loaded metadata for tablet b7a634cdcdbb46aa9d3f79b834ea7d37 (table default.Table_e13a07ffb7754b44a1e224a0cc1e4289 [id=3594565eae314c6e861c681fb61092bf])
I20250628 07:52:47.867519  2740 tablet_loader.cc:96] loaded metadata for tablet bed1ea07e3d44c08adf37ced9dbef85e (table default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6])
I20250628 07:52:47.868649  2740 tablet_loader.cc:96] loaded metadata for tablet bf88267cb80545a5b4b2185732137576 (table default.Table_a90d3fd61b424016a14e3add816d2218 [id=57ab80d617474a998f26259768722a07])
I20250628 07:52:47.869796  2740 tablet_loader.cc:96] loaded metadata for tablet bfc303c4588d44428984d8203fc05185 (table default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4])
I20250628 07:52:47.870968  2740 tablet_loader.cc:96] loaded metadata for tablet c070c23e87594bf5853b2e42aafac7f5 (table default.Table_0b4dcd93c24b4479ad87780460b6afd8 [id=4fe25f3f36eb4a76addfc9f54bb85371])
I20250628 07:52:47.865183  1230 consensus_queue.cc:260] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.873072  1230 ts_tablet_manager.cc:1428] T b69bbcd5fede44ed9c4e1a2c35338d56 P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.012s	user 0.004s	sys 0.000s
I20250628 07:52:47.880872  2740 tablet_loader.cc:96] loaded metadata for tablet c3239c3041c84dfdbc61de4fcb4ea5dd (table default.Table_6541827cef9a442685f4b25c2539eb33 [id=38e154d3c3fc455eb314ff2d8323bb5b])
I20250628 07:52:47.882833  2740 tablet_loader.cc:96] loaded metadata for tablet c426c5a91c7e43658b158fd16b9fa60c (table default.Table_ea4026116e0e482e84f4f48211e0079c [id=b91ffe1c2ac54b0f81f812bf9c13a641])
I20250628 07:52:47.884672  2740 tablet_loader.cc:96] loaded metadata for tablet c4e22ae1341d4262ad8e446315fae0f8 (table default.Table_7a076a51572f4693af02750b6627265e [id=7b1235959a5c4e3b9d4c22edcbba14d6])
I20250628 07:52:47.886579  2740 tablet_loader.cc:96] loaded metadata for tablet c6b42038b74a450796b43fa89d462017 (table default.Table_992ef1fbcdb840c382cc60a218166855 [id=868a11eb38d6484b98187acb9813872f])
I20250628 07:52:47.887722  2740 tablet_loader.cc:96] loaded metadata for tablet c84abdf28d7b4c079d3e6daee9327030 (table default.Table_a011102f62ad4163af1c9aa371b6b29f [id=9c8759743f2c49cf911b12ca926da61f])
I20250628 07:52:47.890179  2740 tablet_loader.cc:96] loaded metadata for tablet ca128104712e4b98b6bea2cb4bd604f2 (table default.Table_168a40d10a3645bdaea410044e99f357 [id=2013562180c34579a816dde8278e9c85])
I20250628 07:52:47.892791  2740 tablet_loader.cc:96] loaded metadata for tablet cc49fd5696e1403e95ea8d60220eafa0 (table default.Table_ee3e5cc9fadf43ef887d84a1d8a735b3 [id=beca5a78fa254e6eb8732613e47278fb])
I20250628 07:52:47.894718  1230 tablet_bootstrap.cc:492] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806: Bootstrap starting.
I20250628 07:52:47.900616  1230 tablet_bootstrap.cc:654] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806: Neither blocks nor log segments found. Creating new log.
I20250628 07:52:47.905923  2740 tablet_loader.cc:96] loaded metadata for tablet d200ea874eca4774a709c592a871e5ba (table default.Table_b6f2044e4f634f6182beb27ccdc55512 [id=11375d9db9b940f2bc7bfcb3c9904cac])
I20250628 07:52:47.908485  2740 tablet_loader.cc:96] loaded metadata for tablet d38887520de2440d8155a19a8954f9cd (table default.Table_9ac8a7257de34accbdf87e0548648d25 [id=a70a1a3bd354432e9ebf6211b0a7a5b7])
I20250628 07:52:47.921511  2740 tablet_loader.cc:96] loaded metadata for tablet d3e486f93a67441fa2da3a72af327503 (table default.Table_27b46453d69840dc832603c0d1ffea0f [id=1a365914a60944d095fa09775cc75aa1])
I20250628 07:52:47.924099  2740 tablet_loader.cc:96] loaded metadata for tablet d96ec557ca9f4008846e8b99610174d5 (table default.Table_239771785031473096dd9da66d0e09d5 [id=cdbc0d29fdf042548e006c7a20e4a34a])
I20250628 07:52:47.925161  2740 tablet_loader.cc:96] loaded metadata for tablet d9e0ea5e300347c7adc8e30b9070976b (table default.Table_478b27f6d90d4be39f296aa259fd334e [id=6c3c849b0c8b437eac1952cc7457d1be])
I20250628 07:52:47.926676  2740 tablet_loader.cc:96] loaded metadata for tablet ddac805491f64b1a95c7cc7d3dba8973 (table default.Table_4bed6916b6b14023b7099f15be8c1edc [id=8d3271dc7fa24639bdc1bf5d4104a459])
I20250628 07:52:47.930500  2740 tablet_loader.cc:96] loaded metadata for tablet e4a1bb1f0adb4a82a199601863983dd7 (table default.Table_affcc9f556b943238389e6f1e205745b [id=070d78e5d6f54ab9a72d680f38247ac1])
I20250628 07:52:47.931617  2740 tablet_loader.cc:96] loaded metadata for tablet e4bf0710bcf145a382516afd4ee09f0b (table default.Table_d65bc763056e4ce181baccde39c92ab9 [id=c3789b7fe8064911bd4095ae69bfd8e5])
I20250628 07:52:47.945544  2740 tablet_loader.cc:96] loaded metadata for tablet e52e15aefe194a61baa7e74616d9bfd3 (table default.Table_5d099833475e4b44b3fda7923e027a37 [id=d49f6e667b6e409c9f8f225158c16236])
I20250628 07:52:47.948267  2740 tablet_loader.cc:96] loaded metadata for tablet e78cb8844a43404f9ee38dec2e532169 (table default.Table_b46689c124f54809961340f6c6b317d5 [id=bc2345b772a947e696627993d69e5e02])
I20250628 07:52:47.950069  2740 tablet_loader.cc:96] loaded metadata for tablet e7d4694dde5b4e47bfb981b46aaacbe5 (table default.Table_3b7ef98c85564dc9b40201e1713b7c55 [id=010e62c6a9a649d78b828d7a7dd2296f])
I20250628 07:52:47.951893  2740 tablet_loader.cc:96] loaded metadata for tablet ec358ab4881d42ad8ffbcf3464ef59a3 (table default.Table_3adf909f0f88430aa21e13480ae65522 [id=75edd41d3e954616b79662475b416941])
I20250628 07:52:47.955284  2740 tablet_loader.cc:96] loaded metadata for tablet ef5d1e5d358e424d818afeeebab3cfa2 (table default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b])
I20250628 07:52:47.956393  2740 tablet_loader.cc:96] loaded metadata for tablet f026cebba0814da3acbc26669d0211d4 (table default.Table_10e3847efdbb4f5ebab4c5e6b8cae9de [id=82f7f172b18f4b59afcc216ff724acc3])
I20250628 07:52:47.957517  2740 tablet_loader.cc:96] loaded metadata for tablet f06829296b994d4c9940bf7fe068ace7 (table default.Table_cc2df6969d8d43ba882f1832a7c4b528 [id=31b9ad86edb24981b8fbe350f2878d9a])
I20250628 07:52:47.959363  2740 tablet_loader.cc:96] loaded metadata for tablet f0c4343648e144c9922d9e6c484abb71 (table default.Table_d8ead90f917d46b9beef6badf9823be1 [id=e658c2419bd748e98d19b83528587ca7])
I20250628 07:52:47.960510  2740 tablet_loader.cc:96] loaded metadata for tablet f0d48652a4da42f3b26b42f65c8db51b (table default.Table_449fb44a73414a90954c0a9ce0904745 [id=9dd118093c2247319e95ff646df9a4f2])
I20250628 07:52:47.964375  1230 tablet_bootstrap.cc:492] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806: No bootstrap required, opened a new log
I20250628 07:52:47.964749  1230 ts_tablet_manager.cc:1397] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806: Time spent bootstrapping tablet: real 0.070s	user 0.015s	sys 0.008s
I20250628 07:52:47.966130  2740 tablet_loader.cc:96] loaded metadata for tablet f2295f958cb34b658d2dfcf836d25e7f (table default.Table_3712d707dd104673a98891d70efe27ea [id=fa958332ea564c108010cff701afd312])
I20250628 07:52:47.966921  1230 raft_consensus.cc:357] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.967499  1230 raft_consensus.cc:383] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:52:47.967752  1230 raft_consensus.cc:738] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a58514e01ea4989809bc2992fb2b806, State: Initialized, Role: FOLLOWER
I20250628 07:52:47.967945  2740 tablet_loader.cc:96] loaded metadata for tablet f49a0de38fe24baab09728a19513878f (table default.Table_49b746aa19f841728b4cf05fa450c090 [id=a12bdfa9daa947988fdabac0b95e453a])
W20250628 07:52:47.968062   534 connection.cc:537] server connection from 127.0.107.130:44463 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:47.968691  1062 connection.cc:537] client connection to 127.0.107.130:35061 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20250628 07:52:47.968367  1230 consensus_queue.cc:260] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82f43cdfab544b17a6a71033983e65d4" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 35061 } } peers { permanent_uuid: "6005c44dfe4848f18a304621cd49e934" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 34311 } } peers { permanent_uuid: "6a58514e01ea4989809bc2992fb2b806" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 37709 } }
I20250628 07:52:47.970347  2740 tablet_loader.cc:96] loaded metadata for tablet f8bce65d32e042ab89ebe168d9657466 (table default.Table_b66cef7061354731bda56976a11f758d [id=380a32248ae74b7697746c5853248303])
I20250628 07:52:47.971495  2740 tablet_loader.cc:96] loaded metadata for tablet f8e389edc2b645a6a5564422adf33641 (table default.Table_bb662e3441d74b638d7c33dc69755650 [id=6e9ea1b8ad2d4977a835f5b0c03d1dc8])
I20250628 07:52:47.971102  1230 ts_tablet_manager.cc:1428] T 9ba579ef503c427295b2ba6185aa3ccb P 6a58514e01ea4989809bc2992fb2b806: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20250628 07:52:47.972590  2740 tablet_loader.cc:96] loaded metadata for tablet f8f42a3a559b448f828d724383043a12 (table default.Table_77748d06f2a54596a43e8a89b5b56011 [id=a32a0dbf0f3a44ff8703768610951a4a])
I20250628 07:52:47.973654  2740 tablet_loader.cc:96] loaded metadata for tablet f91208e5de0b42a5a3e17f26f423ea03 (table default.Table_8dcb758f32dc4345acc05a8e494ea755 [id=cc7adfe6bb724dc79d3fa8c5e4a24ded])
I20250628 07:52:47.975337  2740 tablet_loader.cc:96] loaded metadata for tablet f9a553c69c254fdcaf413b53018bbfb2 (table default.Table_cbbe18b888b7493a8c58ed399a96f972 [id=0e4ec82ab3ee4c9caa6ba2af99acd937])
W20250628 07:52:47.978327   610 connection.cc:537] server connection from 127.0.107.130:58273 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20250628 07:52:47.978549  2740 tablet_loader.cc:96] loaded metadata for tablet fb3d9cc430354e54b82ed7844fd2181c (table default.Table_18695aa15f7843b9a5388e863015aae0 [id=5be853f3c9e74f0a8cebb878f97e4f62])
W20250628 07:52:47.979291  2802 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:52:47.971984 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.130:35061 (local address 127.0.107.131:49387)
0628 07:52:47.974327 (+  2343us) negotiation.cc:107] Waiting for socket to connect
0628 07:52:47.974376 (+    49us) client_negotiation.cc:174] Beginning negotiation
0628 07:52:47.974778 (+   402us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:52:47.978520 (+  3742us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.130:35061: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":1805}
I20250628 07:52:47.980362  2740 tablet_loader.cc:96] loaded metadata for tablet fc4c7770fb564ccd95b3c789b179eecf (table default.Table_5f97f65bd045458e83a0d56856db2b89 [id=8c576a9b2b00448fac7e76983bde42e6])
I20250628 07:52:47.989205   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 1041
I20250628 07:52:47.991892  2740 tablet_loader.cc:96] loaded metadata for tablet fceb8ef4ddb44e7eb11be71a5087bd27 (table default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960])
I20250628 07:52:47.994448  2740 tablet_loader.cc:96] loaded metadata for tablet febd000d51e2480e81766b292304b4f5 (table default.Table_be426ff6be794d35845ebfa8e3357e07 [id=99e9926911304df09ecd802e344295af])
W20250628 07:52:48.003436  2740 catalog_manager.cc:1478] Time spent T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Loading table and tablet metadata into memory: real 1.831s	user 0.511s	sys 0.081s
I20250628 07:52:48.003504   634 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08"
I20250628 07:52:48.003767  2740 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:52:48.004240   634 raft_consensus.cc:2408] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Leader election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 for term 5 because replica has last-logged OpId of term: 4 index: 556, which is greater than that of the candidate, which has last-logged OpId of term: 3 index: 555.
W20250628 07:52:48.004642   634 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54336 (request call id 812) took 1014 ms (client timeout 658 ms). Trace:
W20250628 07:52:48.004966   634 rpcz_store.cc:269] 0628 07:52:46.990236 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:46.990397 (+   161us) service_pool.cc:225] Handling call
0628 07:52:48.004580 (+1014183us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:48.006119   632 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08"
I20250628 07:52:48.006882   632 raft_consensus.cc:2408] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Leader election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 for term 5 because replica has last-logged OpId of term: 4 index: 556, which is greater than that of the candidate, which has last-logged OpId of term: 3 index: 555.
I20250628 07:52:48.007831   633 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:48.008622   633 raft_consensus.cc:2408] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 for term 5 because replica has last-logged OpId of term: 4 index: 556, which is greater than that of the candidate, which has last-logged OpId of term: 3 index: 555.
I20250628 07:52:48.003456   636 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "023ed4cb582a44509bccf42b52c494d7" candidate_term: 5 candidate_status { last_received { term: 3 index: 555 } } ignore_live_leader: false dest_uuid: "992ac4c27fb744deb1edf7765553ef08" is_pre_election: true
I20250628 07:52:48.010061   636 raft_consensus.cc:2408] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 023ed4cb582a44509bccf42b52c494d7 for term 5 because replica has last-logged OpId of term: 4 index: 556, which is greater than that of the candidate, which has last-logged OpId of term: 3 index: 555.
W20250628 07:52:48.010399   636 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54336 (request call id 811) took 1089 ms (client timeout 634 ms). Trace:
W20250628 07:52:48.010700   636 rpcz_store.cc:269] 0628 07:52:46.922798 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:46.922931 (+   133us) service_pool.cc:225] Handling call
0628 07:52:48.010340 (+1087409us) inbound_call.cc:173] Queueing success response
Metrics: {}
I20250628 07:52:48.011943   635 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: Refusing update from remote peer 023ed4cb582a44509bccf42b52c494d7: Log matching property violated. Preceding OpId in replica: term: 4 index: 556. Preceding OpId from leader: term: 5 index: 557. (index mismatch)
I20250628 07:52:48.013976  2769 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 556, Last known committed idx: 555, Time since last communication: 0.000s
I20250628 07:52:48.014525  2769 consensus_queue.cc:1230] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [LEADER]: Peer 992ac4c27fb744deb1edf7765553ef08 log is divergent from this leader: its last log entry 4.556 is not in this leader's log and it has not received anything from this leader yet. Falling back to committed index 555
I20250628 07:52:48.017661   635 pending_rounds.cc:85] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Aborting all ops after (but not including) 555
I20250628 07:52:48.017932   635 pending_rounds.cc:107] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Aborting uncommitted NO_OP operation due to leader change: 4.556
I20250628 07:52:48.018191   635 raft_consensus.cc:2887] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [term 5 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted by new leader
I20250628 07:52:48.049916  2810 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 023ed4cb582a44509bccf42b52c494d7. Latest consensus state: current_term: 5 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:48.050779  2810 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:48.092795   540 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_e3aa2f914796478ea02cdb2a8bbdaf6a [id=85eb6f03ddb44b4c99fa174735d121d6] alter complete (version 2)
I20250628 07:52:48.099598   545 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_1cb8a4472fe64c9bbd6fbd0c4c5278e1 [id=e5c2cd9d2c1b44499a609cc74538868b] alter complete (version 2)
I20250628 07:52:48.101363  2810 sys_catalog.cc:455] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:48.102248  2810 sys_catalog.cc:458] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:52:48.103637   546 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_3a9e65464a9842779a108c92d8bd414e [id=8199977a96084027883e83d166d4533c] alter complete (version 1)
I20250628 07:52:48.104838  2740 catalog_manager.cc:1261] Loaded cluster ID: 21ade90b5448441daee534455946fb6e
I20250628 07:52:48.105774  2817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "023ed4cb582a44509bccf42b52c494d7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5fd67221fa5f424ea0f83e582629d276" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 37209 } } peers { permanent_uuid: "023ed4cb582a44509bccf42b52c494d7" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 33023 } } peers { permanent_uuid: "992ac4c27fb744deb1edf7765553ef08" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 42677 } } }
I20250628 07:52:48.106390  2817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7 [sys.catalog]: This master's current role is: LEADER
I20250628 07:52:48.107225   542 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_cb9c785de9174bbd909ba92bb4377d46 [id=c5e5ff6b3fdc40e4841fe3ba6b78ede4] alter complete (version 1)
I20250628 07:52:48.105070  2740 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
W20250628 07:52:48.121910  2782 op_tracker.cc:284] OpTracker waiting for 2 outstanding ops to complete now for 1.020s
I20250628 07:52:48.122210  2782 op_tracker.cc:286] Dumping up to 50 currently running ops: 
I20250628 07:52:48.122416  2782 op_tracker.cc:290] R-P WriteOp [type=LEADER, start_time=2025-06-28 07:52:48, state=WriteOpState 0x7b5800138600 [op_id=(), ts=7172494000601841664, rows=[MUTATE (int8 entry_type=<redacted>, string entry_id=<redacted>) SET metadata=<redacted>]]]
I20250628 07:52:48.123231  2782 op_tracker.cc:290] R-P WriteOp [type=LEADER, start_time=2025-06-28 07:52:48, state=WriteOpState 0x7b5800079e00 [op_id=(), ts=7172494000586170368, rows=[MUTATE (int8 entry_type=<redacted>, string entry_id=<redacted>) SET metadata=<redacted>]]]
W20250628 07:52:48.127367   546 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 61) took 5013 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.132130   545 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 69) took 6057 ms (client timeout 1000 ms). Trace:
I20250628 07:52:48.135007   542 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_4ba728e60a9b472aa1944ff3caf15795 [id=45f854bb205c451392a6f9976a68fd89] alter complete (version 3)
I20250628 07:52:48.141294   543 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_7d65816a79cc4f9faa8d84ed157ed333 [id=680e55a59eba4e39b0d2def0036257e8] alter complete (version 1)
I20250628 07:52:48.149705   540 catalog_manager.cc:6127] T 00000000000000000000000000000000 P 023ed4cb582a44509bccf42b52c494d7: default.Table_00cd764ff56846b289203b0111658952 [id=d33855cf2b994e0581978cc1922d6960] alter complete (version 1)
W20250628 07:52:48.168201   544 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 71) took 3378 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.185164   544 rpcz_store.cc:269] 0628 07:52:44.790155 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:44.790280 (+   125us) service_pool.cc:225] Handling call
0628 07:52:46.914864 (+2124584us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.914884 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.950271 (+ 35387us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.950290 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.971417 (+ 21127us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.971438 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:47.020437 (+ 48999us) spinlock_profiling.cc:243] Waited 12.5 ms on lock 0x7b5400030300. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8f2dd11 00007f5be9001e19 00007f5be8ff7a3d 00007f5be8fb90eb 00007f5be8fb8ec4 00007f5bea978041 00007f5bea97889f 00007f5bea83fac7 00007f5bea83de90 00007f5bea9400ae 00007f5be8654624
0628 07:52:47.020725 (+   288us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.020746 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:48.168141 (+1147395us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:46.914712 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.914739 (+    27us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.914759 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.914777 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.950116 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.950144 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.950163 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.950181 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.971241 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.971273 (+    32us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.971292 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.971313 (+    21us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.020526 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.020565 (+    39us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.020586 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:47.020606 (+    20us) write_op.cc:335] FINISH: Op aborted
Metrics: {"mutex_wait_us":12325,"reported_tablets":42,"spinlock_wait_cycles":28808064,"child_traces":[["op",{}],["op",{}],["op",{}],["op",{}]]}
I20250628 07:52:48.200009  2740 catalog_manager.cc:1506] Loading token signing keys...
W20250628 07:52:48.172439   543 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 62) took 3938 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.181167   533 consensus_peers.cc:161] SignalRequest failed: Illegal state: peer closed
W20250628 07:52:48.128221   546 rpcz_store.cc:269] 0628 07:52:43.113727 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.128438 (+ 14711us) service_pool.cc:225] Handling call
0628 07:52:43.188848 (+ 60410us) spinlock_profiling.cc:243] Waited 55.6 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:46.905012 (+3716164us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.905032 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.907357 (+  2325us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.907380 (+    23us) write_op.cc:454] Released schema lock
0628 07:52:47.018829 (+111449us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.018853 (+    24us) write_op.cc:454] Released schema lock
0628 07:52:48.127308 (+1108455us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:46.904825 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.904855 (+    30us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.904873 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.904912 (+    39us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.907202 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.907230 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.907248 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.907267 (+    19us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.018608 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.018642 (+    34us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.018660 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:47.018678 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.126347 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:47.126758 (+   411us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:47.126785 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:47.126801 (+    16us) tablet.cc:660] Decoding operations
0628 07:52:47.127056 (+   255us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:47.127093 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:47.127120 (+    27us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:47.127255 (+   135us) tablet.cc:699] Row locks acquired
0628 07:52:47.127270 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:47.127364 (+    94us) write_op.cc:270] Start()
0628 07:52:47.127441 (+    77us) write_op.cc:276] Timestamp: P: 1751097167127314 usec, L: 0
0628 07:52:47.127462 (+    21us) op_driver.cc:348] REPLICATION: starting
0628 07:52:47.127942 (+   480us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.094916 (+966974us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.095835 (+   919us) write_op.cc:301] APPLY: starting
0628 07:52:48.095885 (+    50us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.095968 (+    83us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.096074 (+   106us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.096243 (+   169us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.096369 (+   126us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.096411 (+    42us) write_op.cc:312] APPLY: finished
0628 07:52:48.096586 (+   175us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.100254 (+  3668us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.100304 (+    50us) write_op.cc:454] Released schema lock
0628 07:52:48.100336 (+    32us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":1982,"reported_tablets":28,"spinlock_wait_cycles":127947648,"child_traces":[["op",{}],["op",{}],["op",{}],["op",{"apply.queue_time_us":746,"apply.run_cpu_time_us":1199,"apply.run_wall_time_us":4729,"num_ops":1,"prepare.queue_time_us":46470,"prepare.run_cpu_time_us":1778,"prepare.run_wall_time_us":1777,"replication_time_us":967339}]]}
W20250628 07:52:48.215392   534 connection.cc:441] server connection from 127.0.107.130:44463 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 61) could send its response
W20250628 07:52:48.191555   547 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 79) took 2954 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.216591   547 rpcz_store.cc:269] 0628 07:52:45.237275 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:45.237387 (+   112us) service_pool.cc:225] Handling call
0628 07:52:46.883339 (+1645952us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.883359 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.936466 (+ 53107us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.936687 (+   221us) write_op.cc:454] Released schema lock
0628 07:52:48.191495 (+1254808us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:46.883164 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.883197 (+    33us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.883222 (+    25us) write_op.cc:454] Released schema lock
0628 07:52:46.883240 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.934709 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.934952 (+   243us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.935164 (+   212us) write_op.cc:454] Released schema lock
0628 07:52:46.935188 (+    24us) write_op.cc:335] FINISH: Op aborted
Metrics: {"mutex_wait_us":29104,"reported_tablets":53,"child_traces":[["op",{}],["op",{}]]}
W20250628 07:52:48.205024   543 rpcz_store.cc:269] 0628 07:52:44.233887 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:44.234011 (+   124us) service_pool.cc:225] Handling call
0628 07:52:46.910976 (+2676965us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.910998 (+    22us) write_op.cc:454] Released schema lock
0628 07:52:46.915482 (+  4484us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.915503 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:46.952940 (+ 37437us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.952959 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:47.032564 (+ 79605us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.032583 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:48.172380 (+1139797us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:45.248873 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:45.249250 (+   377us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:45.249275 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:45.249286 (+    11us) tablet.cc:660] Decoding operations
0628 07:52:45.249592 (+   306us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:45.249634 (+    42us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:45.249655 (+    21us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:45.249888 (+   233us) tablet.cc:699] Row locks acquired
0628 07:52:45.249901 (+    13us) write_op.cc:260] PREPARE: finished
0628 07:52:45.250006 (+   105us) write_op.cc:270] Start()
0628 07:52:45.250085 (+    79us) write_op.cc:276] Timestamp: P: 1751097165249927 usec, L: 0
0628 07:52:45.250101 (+    16us) op_driver.cc:348] REPLICATION: starting
0628 07:52:45.250583 (+   482us) log.cc:844] Serialized 913 byte log entry
0628 07:52:45.261914 (+ 11331us) op_driver.cc:464] REPLICATION: finished
0628 07:52:45.262278 (+   364us) write_op.cc:301] APPLY: starting
0628 07:52:45.262345 (+    67us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:45.262486 (+   141us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:45.262597 (+   111us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:45.263093 (+   496us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:45.263275 (+   182us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:45.263315 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:45.263506 (+   191us) log.cc:844] Serialized 29 byte log entry
0628 07:52:45.263664 (+   158us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:45.263767 (+   103us) write_op.cc:454] Released schema lock
0628 07:52:45.263812 (+    45us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:45.264098 (+   286us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:45.264130 (+    32us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:46.910813 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.910846 (+    33us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.910866 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.910883 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.912955 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.912985 (+    30us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.913005 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:46.913023 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.952786 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.952817 (+    31us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.952835 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.952853 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.032405 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.032435 (+    30us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.032453 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:47.032471 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:48.114673 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:48.115093 (+   420us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:48.115116 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:48.115131 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:48.115394 (+   263us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:48.115428 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:48.115452 (+    24us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:48.115608 (+   156us) tablet.cc:699] Row locks acquired
0628 07:52:48.115624 (+    16us) write_op.cc:260] PREPARE: finished
0628 07:52:48.115732 (+   108us) write_op.cc:270] Start()
0628 07:52:48.115813 (+    81us) write_op.cc:276] Timestamp: P: 1751097168115684 usec, L: 0
0628 07:52:48.115831 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:48.116279 (+   448us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.138268 (+ 21989us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.138554 (+   286us) write_op.cc:301] APPLY: starting
0628 07:52:48.138601 (+    47us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.138673 (+    72us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.138755 (+    82us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.138928 (+   173us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.139046 (+   118us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.139086 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:48.139232 (+   146us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.139353 (+   121us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.139405 (+    52us) write_op.cc:454] Released schema lock
0628 07:52:48.139433 (+    28us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":53454,"reported_tablets":34,"child_traces":[["op",{"apply.queue_time_us":208,"apply.run_cpu_time_us":1783,"apply.run_wall_time_us":1782,"num_ops":2,"prepare.queue_time_us":9819,"prepare.run_cpu_time_us":1982,"prepare.run_wall_time_us":1982,"raft.queue_time_us":1188,"raft.run_cpu_time_us":1064,"raft.run_wall_time_us":1064,"replication_time_us":11703}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{"apply.queue_time_us":168,"apply.run_cpu_time_us":1060,"apply.run_wall_time_us":1060,"num_ops":1,"prepare.queue_time_us":297,"prepare.run_cpu_time_us":1798,"prepare.run_wall_time_us":2714,"replication_time_us":22353}]]}
W20250628 07:52:48.245461   534 connection.cc:441] server connection from 127.0.107.130:44463 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 62) could send its response
W20250628 07:52:48.132505   545 rpcz_store.cc:269] 0628 07:52:42.074357 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.273196 (+198839us) service_pool.cc:225] Handling call
0628 07:52:42.515165 (+241969us) spinlock_profiling.cc:243] Waited 77.2 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:46.868978 (+4353813us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.868999 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:46.900175 (+ 31176us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.900193 (+    18us) write_op.cc:454] Released schema lock
0628 07:52:46.945351 (+ 45158us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.945368 (+    17us) write_op.cc:454] Released schema lock
0628 07:52:47.023304 (+ 77936us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.023321 (+    17us) write_op.cc:454] Released schema lock
0628 07:52:47.043624 (+ 20303us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.043648 (+    24us) write_op.cc:454] Released schema lock
0628 07:52:48.132070 (+1088422us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:44.043684 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:44.044036 (+   352us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:44.044059 (+    23us) write_op.cc:435] Acquired schema lock
0628 07:52:44.044071 (+    12us) tablet.cc:660] Decoding operations
0628 07:52:44.044302 (+   231us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:44.044339 (+    37us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:44.044359 (+    20us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:44.044495 (+   136us) tablet.cc:699] Row locks acquired
0628 07:52:44.044507 (+    12us) write_op.cc:260] PREPARE: finished
0628 07:52:44.044617 (+   110us) write_op.cc:270] Start()
0628 07:52:44.044690 (+    73us) write_op.cc:276] Timestamp: P: 1751097164044566 usec, L: 0
0628 07:52:44.044705 (+    15us) op_driver.cc:348] REPLICATION: starting
0628 07:52:44.045177 (+   472us) log.cc:844] Serialized 533 byte log entry
0628 07:52:44.052710 (+  7533us) op_driver.cc:464] REPLICATION: finished
0628 07:52:44.053050 (+   340us) write_op.cc:301] APPLY: starting
0628 07:52:44.053125 (+    75us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:44.053225 (+   100us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:44.053318 (+    93us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:44.053530 (+   212us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:44.053667 (+   137us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:44.053726 (+    59us) write_op.cc:312] APPLY: finished
0628 07:52:44.053923 (+   197us) log.cc:844] Serialized 23 byte log entry
0628 07:52:44.054077 (+   154us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:44.054152 (+    75us) write_op.cc:454] Released schema lock
0628 07:52:44.054190 (+    38us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:44.054408 (+   218us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:44.054430 (+    22us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:46.868769 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.868817 (+    48us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.868836 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.868853 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.900015 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.900043 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.900062 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.900081 (+    19us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.945191 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.945218 (+    27us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.945237 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:46.945255 (+    18us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.023128 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.023155 (+    27us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.023174 (+    19us) write_op.cc:454] Released schema lock
0628 07:52:47.023194 (+    20us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.036966 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.037011 (+    45us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.037028 (+    17us) write_op.cc:454] Released schema lock
0628 07:52:47.037047 (+    19us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.110047 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:47.110555 (+   508us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:47.110588 (+    33us) write_op.cc:435] Acquired schema lock
0628 07:52:47.110608 (+    20us) tablet.cc:660] Decoding operations
0628 07:52:47.110881 (+   273us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:47.110917 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:47.110945 (+    28us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:47.111079 (+   134us) tablet.cc:699] Row locks acquired
0628 07:52:47.111095 (+    16us) write_op.cc:260] PREPARE: finished
0628 07:52:47.111203 (+   108us) write_op.cc:270] Start()
0628 07:52:47.111283 (+    80us) write_op.cc:276] Timestamp: P: 1751097167111156 usec, L: 0
0628 07:52:47.111301 (+    18us) op_driver.cc:348] REPLICATION: starting
0628 07:52:47.111795 (+   494us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.084237 (+972442us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.090050 (+  5813us) write_op.cc:301] APPLY: starting
0628 07:52:48.090105 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.090187 (+    82us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.090297 (+   110us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.090509 (+   212us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.090644 (+   135us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.090687 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:48.097104 (+  6417us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.097626 (+   522us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.097675 (+    49us) write_op.cc:454] Released schema lock
0628 07:52:48.097706 (+    31us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":44396,"reported_tablets":39,"spinlock_wait_cycles":177604352,"child_traces":[["op",{"apply.queue_time_us":183,"apply.run_cpu_time_us":1323,"apply.run_wall_time_us":1322,"num_ops":1,"prepare.queue_time_us":116,"prepare.run_cpu_time_us":1764,"prepare.run_wall_time_us":7291,"raft.queue_time_us":80,"raft.run_cpu_time_us":589,"raft.run_wall_time_us":589,"replication_time_us":7916}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{}],["op",{"apply.queue_time_us":5661,"apply.run_cpu_time_us":1276,"apply.run_wall_time_us":7855,"num_ops":1,"prepare.queue_time_us":31500,"prepare.run_cpu_time_us":1937,"prepare.run_wall_time_us":1937,"replication_time_us":972855,"thread_start_us":7161,"threads_started":1}]]}
W20250628 07:52:48.202121   542 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 77) took 5848 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.236778   531 connection.cc:537] server connection from 127.0.107.131:60285 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:48.241273   533 connection.cc:441] server connection from 127.0.107.129:43357 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 71) could send its response
W20250628 07:52:48.248188   533 connection.cc:441] server connection from 127.0.107.129:43357 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.129:43357 (request call id 69) could send its response
W20250628 07:52:48.249639   533 connection.cc:537] client connection to 127.0.107.131:37709 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:52:48.194924   540 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 78) took 4465 ms (client timeout 1000 ms). Trace:
I20250628 07:52:48.252759   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 1384
I20250628 07:52:48.259459  2740 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 992ac4c27fb744deb1edf7765553ef08: Loaded TSK: 0
I20250628 07:52:48.260545  2740 catalog_manager.cc:1516] Initializing in-progress tserver states...
W20250628 07:52:48.264712   538 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 63) took 2502 ms (client timeout 1000 ms). Trace:
W20250628 07:52:48.268484   538 rpcz_store.cc:269] 0628 07:52:45.762259 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:45.762410 (+   151us) service_pool.cc:225] Handling call
0628 07:52:48.264643 (+2502233us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:47.112079 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:47.112457 (+   378us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:47.112482 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:47.112497 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:47.112873 (+   376us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:47.112912 (+    39us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:47.112938 (+    26us) tablet.cc:683] Acquiring locks for 2 operations
0628 07:52:47.119896 (+  6958us) tablet.cc:699] Row locks acquired
0628 07:52:47.119922 (+    26us) write_op.cc:260] PREPARE: finished
0628 07:52:47.121016 (+  1094us) write_op.cc:270] Start()
0628 07:52:47.121612 (+   596us) write_op.cc:276] Timestamp: P: 1751097167120765 usec, L: 0
0628 07:52:47.121634 (+    22us) op_driver.cc:348] REPLICATION: starting
0628 07:52:47.126034 (+  4400us) log.cc:844] Serialized 913 byte log entry
0628 07:52:48.091732 (+965698us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.092752 (+  1020us) write_op.cc:301] APPLY: starting
0628 07:52:48.092803 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.092907 (+   104us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.092989 (+    82us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.093261 (+   272us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.093392 (+   131us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
0628 07:52:48.093435 (+    43us) write_op.cc:312] APPLY: finished
0628 07:52:48.093593 (+   158us) log.cc:844] Serialized 29 byte log entry
0628 07:52:48.099293 (+  5700us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.099352 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:48.099381 (+    29us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":23132,"reported_tablets":37,"child_traces":[["op",{"apply.queue_time_us":906,"apply.run_cpu_time_us":1243,"apply.run_wall_time_us":6807,"num_ops":2,"prepare.queue_time_us":33449,"prepare.run_cpu_time_us":2360,"prepare.run_wall_time_us":14143,"replication_time_us":969842,"thread_start_us":2848,"threads_started":1}]]}
W20250628 07:52:48.274818   534 connection.cc:441] server connection from 127.0.107.130:44463 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.130:44463 (request call id 63) could send its response
W20250628 07:52:48.256170   542 rpcz_store.cc:269] 0628 07:52:42.354134 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:42.446076 (+ 91942us) service_pool.cc:225] Handling call
0628 07:52:42.465253 (+ 19177us) spinlock_profiling.cc:243] Waited 7.95 ms on lock 0x7b5400030554. stack: 00007f5be1c114b3 00007f5be1c113b0 00007f5be178d574 00000000004fa174 00000000004fa109 00007f5be8b13bc0 00007f5be8b97c6a 00007f5bea846fac 00007f5bea93f7ee 00007f5be8654624 00007f5be86545b2 00007f5be86544e1 00007f5be865445c 00007f5be8653712 00007f5be6957dcc 00007f5be6957206
0628 07:52:46.976677 (+4511424us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.976697 (+    20us) write_op.cc:454] Released schema lock
0628 07:52:47.061318 (+ 84621us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.061339 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:48.202060 (+1140721us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:46.976523 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.976551 (+    28us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.976568 (+    17us) write_op.cc:454] Released schema lock
0628 07:52:46.976585 (+    17us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.053058 (+     0us) op_driver.cc:389] HandleFailure(Service unavailable: leader is not yet ready)
0628 07:52:47.061134 (+  8076us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.061161 (+    27us) write_op.cc:454] Released schema lock
0628 07:52:47.061187 (+    26us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.128235 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:47.128631 (+   396us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:47.128656 (+    25us) write_op.cc:435] Acquired schema lock
0628 07:52:47.128672 (+    16us) tablet.cc:660] Decoding operations
0628 07:52:47.128937 (+   265us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:47.128971 (+    34us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:47.128998 (+    27us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:47.141494 (+ 12496us) tablet.cc:699] Row locks acquired
0628 07:52:47.141624 (+   130us) write_op.cc:260] PREPARE: finished
0628 07:52:47.142051 (+   427us) write_op.cc:270] Start()
0628 07:52:47.142357 (+   306us) write_op.cc:276] Timestamp: P: 1751097167141892 usec, L: 0
0628 07:52:47.142480 (+   123us) op_driver.cc:348] REPLICATION: starting
0628 07:52:47.144146 (+  1666us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.095158 (+951012us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.098001 (+  2843us) write_op.cc:301] APPLY: starting
0628 07:52:48.098048 (+    47us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.098127 (+    79us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.098220 (+    93us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.098361 (+   141us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.098486 (+   125us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.098545 (+    59us) write_op.cc:312] APPLY: finished
0628 07:52:48.098679 (+   134us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.106889 (+  8210us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.106945 (+    56us) write_op.cc:454] Released schema lock
0628 07:52:48.106974 (+    29us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:48.110700 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:48.111187 (+   487us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:48.111214 (+    27us) write_op.cc:435] Acquired schema lock
0628 07:52:48.111232 (+    18us) tablet.cc:660] Decoding operations
0628 07:52:48.111543 (+   311us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:48.111586 (+    43us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:48.111616 (+    30us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:48.111776 (+   160us) tablet.cc:699] Row locks acquired
0628 07:52:48.111791 (+    15us) write_op.cc:260] PREPARE: finished
0628 07:52:48.111903 (+   112us) write_op.cc:270] Start()
0628 07:52:48.111987 (+    84us) write_op.cc:276] Timestamp: P: 1751097168111858 usec, L: 0
0628 07:52:48.112006 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:48.112556 (+   550us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.127930 (+ 15374us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.128595 (+   665us) write_op.cc:301] APPLY: starting
0628 07:52:48.128646 (+    51us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.128725 (+    79us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.128813 (+    88us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.128970 (+   157us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.129110 (+   140us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.129155 (+    45us) write_op.cc:312] APPLY: finished
0628 07:52:48.129309 (+   154us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.129973 (+   664us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.130027 (+    54us) write_op.cc:454] Released schema lock
0628 07:52:48.130054 (+    27us) write_op.cc:341] FINISH: Updating metrics
Metrics: {"mutex_wait_us":19370,"reported_tablets":50,"spinlock_wait_cycles":18277120,"child_traces":[["op",{}],["op",{}],["op",{"apply.queue_time_us":2738,"apply.run_cpu_time_us":1083,"apply.run_wall_time_us":9141,"num_ops":1,"prepare.queue_time_us":39768,"prepare.run_cpu_time_us":2193,"prepare.run_wall_time_us":16503,"replication_time_us":952591,"thread_start_us":9884,"threads_started":1}],["op",{"apply.queue_time_us":535,"apply.run_cpu_time_us":1087,"apply.run_wall_time_us":1624,"num_ops":1,"prepare.queue_time_us":83,"prepare.run_cpu_time_us":2153,"prepare.run_wall_time_us":2154,"raft.queue_time_us":1123,"raft.run_cpu_time_us":592,"raft.run_wall_time_us":592,"replication_time_us":15836}]]}
W20250628 07:52:48.277530   531 connection.cc:441] server connection from 127.0.107.131:60285 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 77) could send its response
W20250628 07:52:48.251437   540 rpcz_store.cc:269] 0628 07:52:43.729188 (+     0us) service_pool.cc:168] Inserting onto call queue
0628 07:52:43.729305 (+   117us) service_pool.cc:225] Handling call
0628 07:52:46.876513 (+3147208us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.876534 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:46.973075 (+ 96541us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:47.001370 (+ 28295us) write_op.cc:454] Released schema lock
0628 07:52:48.194868 (+1193498us) inbound_call.cc:173] Queueing success response
Related trace 'op':
0628 07:52:45.000189 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:45.000687 (+   498us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:45.000719 (+    32us) write_op.cc:435] Acquired schema lock
0628 07:52:45.000734 (+    15us) tablet.cc:660] Decoding operations
0628 07:52:45.001029 (+   295us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:45.001080 (+    51us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:45.001136 (+    56us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:45.001305 (+   169us) tablet.cc:699] Row locks acquired
0628 07:52:45.001334 (+    29us) write_op.cc:260] PREPARE: finished
0628 07:52:45.001452 (+   118us) write_op.cc:270] Start()
0628 07:52:45.001558 (+   106us) write_op.cc:276] Timestamp: P: 1751097165001392 usec, L: 0
0628 07:52:45.001580 (+    22us) op_driver.cc:348] REPLICATION: starting
0628 07:52:45.002089 (+   509us) log.cc:844] Serialized 533 byte log entry
0628 07:52:45.019510 (+ 17421us) op_driver.cc:464] REPLICATION: finished
0628 07:52:45.019887 (+   377us) write_op.cc:301] APPLY: starting
0628 07:52:45.019957 (+    70us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:45.020069 (+   112us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:45.020189 (+   120us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:45.020384 (+   195us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:45.020567 (+   183us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:45.020626 (+    59us) write_op.cc:312] APPLY: finished
0628 07:52:45.020817 (+   191us) log.cc:844] Serialized 23 byte log entry
0628 07:52:45.020981 (+   164us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:45.021061 (+    80us) write_op.cc:454] Released schema lock
0628 07:52:45.021116 (+    55us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:45.021350 (+   234us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:45.021373 (+    23us) write_op.cc:454] Released schema lock
Related trace 'op':
0628 07:52:46.876316 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.876351 (+    35us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.876372 (+    21us) write_op.cc:454] Released schema lock
0628 07:52:46.876393 (+    21us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:46.972764 (+     0us) op_driver.cc:389] HandleFailure(Illegal state: replica 023ed4cb582a44509bccf42b52c494d7 is not leader of this config: current role FOLLOWER)
0628 07:52:46.972823 (+    59us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:46.972869 (+    46us) write_op.cc:454] Released schema lock
0628 07:52:46.972888 (+    19us) write_op.cc:335] FINISH: Op aborted
Related trace 'op':
0628 07:52:47.098157 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:47.098585 (+   428us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:47.098611 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:47.098621 (+    10us) tablet.cc:660] Decoding operations
0628 07:52:47.098892 (+   271us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:47.098952 (+    60us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:47.098976 (+    24us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:47.099140 (+   164us) tablet.cc:699] Row locks acquired
0628 07:52:47.099150 (+    10us) write_op.cc:260] PREPARE: finished
0628 07:52:47.099286 (+   136us) write_op.cc:270] Start()
0628 07:52:47.099369 (+    83us) write_op.cc:276] Timestamp: P: 1751097167099232 usec, L: 0
0628 07:52:47.099388 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:47.099938 (+   550us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.075283 (+975345us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.082818 (+  7535us) write_op.cc:301] APPLY: starting
0628 07:52:48.082873 (+    55us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.082955 (+    82us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.083047 (+    92us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.083204 (+   157us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.083363 (+   159us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.083403 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:48.083583 (+   180us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.092350 (+  8767us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.092409 (+    59us) write_op.cc:454] Released schema lock
0628 07:52:48.092434 (+    25us) write_op.cc:341] FINISH: Updating metrics
Related trace 'op':
0628 07:52:48.125169 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
0628 07:52:48.125615 (+   446us) write_op.cc:432] Acquiring schema lock in shared mode
0628 07:52:48.125641 (+    26us) write_op.cc:435] Acquired schema lock
0628 07:52:48.125658 (+    17us) tablet.cc:660] Decoding operations
0628 07:52:48.125922 (+   264us) write_op.cc:620] Acquiring the partition lock for write op
0628 07:52:48.125958 (+    36us) write_op.cc:641] Partition lock acquired for write op
0628 07:52:48.125980 (+    22us) tablet.cc:683] Acquiring locks for 1 operations
0628 07:52:48.126122 (+   142us) tablet.cc:699] Row locks acquired
0628 07:52:48.126138 (+    16us) write_op.cc:260] PREPARE: finished
0628 07:52:48.126255 (+   117us) write_op.cc:270] Start()
0628 07:52:48.126332 (+    77us) write_op.cc:276] Timestamp: P: 1751097168126196 usec, L: 0
0628 07:52:48.126351 (+    19us) op_driver.cc:348] REPLICATION: starting
0628 07:52:48.126821 (+   470us) log.cc:844] Serialized 356 byte log entry
0628 07:52:48.148030 (+ 21209us) op_driver.cc:464] REPLICATION: finished
0628 07:52:48.148449 (+   419us) write_op.cc:301] APPLY: starting
0628 07:52:48.148507 (+    58us) tablet.cc:1365] starting BulkCheckPresence
0628 07:52:48.148601 (+    94us) tablet.cc:1368] finished BulkCheckPresence
0628 07:52:48.148689 (+    88us) tablet.cc:1370] starting ApplyRowOperation cycle
0628 07:52:48.148840 (+   151us) tablet.cc:1381] finished ApplyRowOperation cycle
0628 07:52:48.148991 (+   151us) tablet_metrics.cc:563] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
0628 07:52:48.149031 (+    40us) write_op.cc:312] APPLY: finished
0628 07:52:48.149226 (+   195us) log.cc:844] Serialized 23 byte log entry
0628 07:52:48.149346 (+   120us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.149401 (+    55us) write_op.cc:454] Released schema lock
0628 07:52:48.149434 (+    33us) write_op.cc:341] FINISH: Updating metrics
0628 07:52:48.149637 (+   203us) write_op.cc:489] Releasing partition, row and schema locks
0628 07:52:48.149657 (+    20us) write_op.cc:454] Released schema lock
Metrics: {"mutex_wait_us":51589,"reported_tablets":52,"child_traces":[["op",{"apply.queue_time_us":201,"apply.run_cpu_time_us":1437,"apply.run_wall_time_us":1436,"num_ops":1,"prepare.queue_time_us":363,"prepare.run_cpu_time_us":2231,"prepare.run_wall_time_us":2231,"raft.queue_time_us":1530,"raft.run_cpu_time_us":732,"raft.run_wall_time_us":733,"replication_time_us":17807}],["op",{}],["op",{}],["op",{"apply.queue_time_us":7375,"apply.run_cpu_time_us":1223,"apply.run_wall_time_us":9839,"num_ops":1,"prepare.queue_time_us":30982,"prepare.run_cpu_time_us":2207,"prepare.run_wall_time_us":11730,"raft.queue_time_us":6239,"raft.run_cpu_time_us":944,"raft.run_wall_time_us":1806,"replication_time_us":975779,"thread_start_us":41713,"threads_started":2}],["op",{"apply.queue_time_us":284,"apply.run_cpu_time_us":1165,"apply.run_wall_time_us":1157,"num_ops":1,"prepare.queue_time_us":94,"prepare.run_cpu_time_us":1876,"prepare.run_wall_time_us":4724,"replication_time_us":21590}]]}
W20250628 07:52:48.282533   531 connection.cc:441] server connection from 127.0.107.131:60285 torn down before Call kudu.master.MasterService.TSHeartbeat from 127.0.107.131:60285 (request call id 78) could send its response
I20250628 07:52:48.330962   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 509
I20250628 07:52:48.374125   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 584
2025-06-28T07:52:48Z chronyd exiting
I20250628 07:52:48.430054   430 master-stress-test.cc:585] Tables created: 236
I20250628 07:52:48.430241   430 master-stress-test.cc:586] Tables altered: 94
I20250628 07:52:48.430359   430 master-stress-test.cc:587] Tables deleted: 91
I20250628 07:52:48.430514   430 master-stress-test.cc:588] Tablets replaced: 35
I20250628 07:52:48.430627   430 master-stress-test.cc:589] Masters restarted: 1
[       OK ] CatalogManagerConfigurations/MasterStressTest.Test/0 (52030 ms)
[ RUN      ] CatalogManagerConfigurations/MasterStressTest.Test/3
I20250628 07:52:48.532081   430 test_util.cc:276] Using random seed: 347247194
2025-06-28T07:52:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
2025-06-28T07:52:48Z Disabled control of system clock
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/security.properties
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/tmp/dist-test-taskVIvbNg/thirdparty/src/hive-3.1.2/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/tmp/dist-test-taskVIvbNg/thirdparty/src/hadoop-3.4.1/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Metastore connection URL:	 jdbc:derby:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/metadb;create=true
Metastore Connection Driver :	 org.apache.derby.jdbc.EmbeddedDriver
Metastore connection User:	 APP
Starting metastore schema initialization to 3.1.0
Initialization script hive-schema-3.1.0.derby.sql

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 

 
 
 
 

 
 
 
 


 
 
 

 
 
 


 
 
 
 
 
 
 


 
 
 

 
 
 

 
 

 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 


 
 
 
 
 
 



 
 
 
 
 


 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 


Initialization script completed
schemaTool completed
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/security.properties
2025-06-28 07:52:54: Starting Hive Metastore Server
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/security.properties
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/tmp/dist-test-taskVIvbNg/thirdparty/src/hive-3.1.2/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/tmp/dist-test-taskVIvbNg/thirdparty/src/hadoop-3.4.1/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Starting hive metastore on port 0
I20250628 07:53:02.638386   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:46841
--webserver_interface=127.0.107.190
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--hive_metastore_uris=thrift://127.0.0.1:34709
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20250628 07:53:02.932502  3064 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:02.933127  3064 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:02.933632  3064 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:02.964268  3064 flags.cc:425] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20250628 07:53:02.964570  3064 flags.cc:425] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20250628 07:53:02.964829  3064 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:53:02.965075  3064 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:02.965356  3064 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:53:02.965600  3064 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:53:03.000746  3064 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
--hive_metastore_uris=thrift://127.0.0.1:34709
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:46841
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.107.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:03.002148  3064 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:03.003731  3064 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:03.018708  3070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:03.019018  3071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:03.020043  3064 server_base.cc:1048] running on GCE node
W20250628 07:53:03.020223  3073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:04.191315  3064 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:04.193944  3064 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:04.195302  3064 hybrid_clock.cc:648] HybridClock initialized: now 1751097184195271 us; error 44 us; skew 500 ppm
I20250628 07:53:04.196096  3064 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:04.202440  3064 webserver.cc:469] Webserver started at http://127.0.107.190:38719/ using document root <none> and password file <none>
I20250628 07:53:04.203357  3064 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:04.203590  3064 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:04.204046  3064 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:04.208477  3064 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/instance:
uuid: "61070f3974844b798ec0e9a7c3c68f2f"
format_stamp: "Formatted at 2025-06-28 07:53:04 on dist-test-slave-kfnw"
I20250628 07:53:04.209628  3064 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal/instance:
uuid: "61070f3974844b798ec0e9a7c3c68f2f"
format_stamp: "Formatted at 2025-06-28 07:53:04 on dist-test-slave-kfnw"
I20250628 07:53:04.217192  3064 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.000s
I20250628 07:53:04.222909  3080 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:04.223968  3064 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
I20250628 07:53:04.224278  3064 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
uuid: "61070f3974844b798ec0e9a7c3c68f2f"
format_stamp: "Formatted at 2025-06-28 07:53:04 on dist-test-slave-kfnw"
I20250628 07:53:04.224604  3064 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:04.277129  3064 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:04.278632  3064 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:04.279084  3064 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:04.352020  3064 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.190:46841
I20250628 07:53:04.352102  3131 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.190:46841 every 8 connection(s)
I20250628 07:53:04.354650  3064 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
I20250628 07:53:04.359678  3132 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:04.362108   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3064
I20250628 07:53:04.362490   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal/instance
I20250628 07:53:04.364851   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.189:42143
--webserver_interface=127.0.107.189
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--hive_metastore_uris=thrift://127.0.0.1:34709
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:53:04.370126  3132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:04.395188  3132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:53:04.397446  3083 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:04.401546  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:04.451121  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 1
W20250628 07:53:04.455022  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:04.523538  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 2
W20250628 07:53:04.527398  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:04.618886  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 3
W20250628 07:53:04.622684  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:04.664445  3133 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:04.664987  3133 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:04.665460  3133 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:04.698087  3133 flags.cc:425] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20250628 07:53:04.698318  3133 flags.cc:425] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20250628 07:53:04.698532  3133 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:53:04.698731  3133 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:04.698928  3133 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:53:04.699119  3133 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:53:04.733290  3133 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal
--hive_metastore_uris=thrift://127.0.0.1:34709
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.189:42143
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
--webserver_interface=127.0.107.189
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:04.734567  3133 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:04.736063  3133 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:04.750033  3145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:04.766157  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 4
W20250628 07:53:04.771543  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:05.070940  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 5
W20250628 07:53:05.076052  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:05.623452  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 6
W20250628 07:53:05.632329  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:04.750155  3143 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:04.750942  3142 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:05.901357  3144 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20250628 07:53:05.901472  3133 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:53:05.905721  3133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:05.908176  3133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:05.909503  3133 hybrid_clock.cc:648] HybridClock initialized: now 1751097185909463 us; error 55 us; skew 500 ppm
I20250628 07:53:05.910267  3133 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:05.921314  3133 webserver.cc:469] Webserver started at http://127.0.107.189:37773/ using document root <none> and password file <none>
I20250628 07:53:05.922250  3133 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:05.922472  3133 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:05.922924  3133 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:05.927206  3133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data/instance:
uuid: "1ed16b61a11544a8968b81868339a936"
format_stamp: "Formatted at 2025-06-28 07:53:05 on dist-test-slave-kfnw"
I20250628 07:53:05.928232  3133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal/instance:
uuid: "1ed16b61a11544a8968b81868339a936"
format_stamp: "Formatted at 2025-06-28 07:53:05 on dist-test-slave-kfnw"
I20250628 07:53:05.935189  3133 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.009s	sys 0.000s
I20250628 07:53:05.940650  3154 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:05.941704  3133 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
I20250628 07:53:05.942024  3133 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal
uuid: "1ed16b61a11544a8968b81868339a936"
format_stamp: "Formatted at 2025-06-28 07:53:05 on dist-test-slave-kfnw"
I20250628 07:53:05.942358  3133 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:05.998272  3133 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:05.999704  3133 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:06.000118  3133 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:06.069664  3133 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.189:42143
I20250628 07:53:06.069747  3205 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.189:42143 every 8 connection(s)
I20250628 07:53:06.072312  3133 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/data/info.pb
I20250628 07:53:06.077198  3206 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:06.082660   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3133
I20250628 07:53:06.083046   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-1/wal/instance
I20250628 07:53:06.085366   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:36455
--webserver_interface=127.0.107.188
--webserver_port=0
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--hive_metastore_uris=thrift://127.0.0.1:34709
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:53:06.087364  3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:06.111639  3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:06.124456  3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:53:06.126386  3157 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
W20250628 07:53:06.130111  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.179656  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 1
W20250628 07:53:06.183444  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.251935  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 2
W20250628 07:53:06.255749  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.347100  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 3
W20250628 07:53:06.352063  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
W20250628 07:53:06.386070  3207 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:06.386683  3207 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:06.387132  3207 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:06.417811  3207 flags.cc:425] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20250628 07:53:06.418143  3207 flags.cc:425] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20250628 07:53:06.418407  3207 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:53:06.418641  3207 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:06.418875  3207 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:53:06.419103  3207 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:53:06.454280  3207 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
--hive_metastore_uris=thrift://127.0.0.1:34709
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:36455
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.107.188
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:06.455677  3207 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:06.457319  3207 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:06.471954  3218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:06.495517  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 4
W20250628 07:53:06.499925  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.692982  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } attempt: 7
I20250628 07:53:06.705117  3132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } has no permanent_uuid. Determining permanent_uuid...
W20250628 07:53:06.710913  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.769258  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 1
W20250628 07:53:06.773571  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.799331  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 5
W20250628 07:53:06.803848  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.854960  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 2
W20250628 07:53:06.859504  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:06.944886  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 3
W20250628 07:53:06.949352  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:07.089725  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 4
W20250628 07:53:07.093464  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:07.351188  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 6
W20250628 07:53:07.359265  3206 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:07.376828  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 5
W20250628 07:53:07.380949  3132 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
W20250628 07:53:06.472153  3220 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:06.473233  3207 server_base.cc:1048] running on GCE node
W20250628 07:53:06.472601  3217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:07.657239  3207 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:07.660305  3207 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:07.661710  3207 hybrid_clock.cc:648] HybridClock initialized: now 1751097187661675 us; error 52 us; skew 500 ppm
I20250628 07:53:07.662518  3207 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:07.669272  3207 webserver.cc:469] Webserver started at http://127.0.107.188:42931/ using document root <none> and password file <none>
I20250628 07:53:07.670148  3207 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:07.670367  3207 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:07.670821  3207 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:07.675062  3207 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/instance:
uuid: "fe360a66f1a24378a030a69807008e4d"
format_stamp: "Formatted at 2025-06-28 07:53:07 on dist-test-slave-kfnw"
I20250628 07:53:07.676069  3207 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal/instance:
uuid: "fe360a66f1a24378a030a69807008e4d"
format_stamp: "Formatted at 2025-06-28 07:53:07 on dist-test-slave-kfnw"
I20250628 07:53:07.682999  3207 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.004s
I20250628 07:53:07.688091  3233 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:07.689052  3207 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
I20250628 07:53:07.689376  3207 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
uuid: "fe360a66f1a24378a030a69807008e4d"
format_stamp: "Formatted at 2025-06-28 07:53:07 on dist-test-slave-kfnw"
I20250628 07:53:07.689769  3207 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:07.754333  3207 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:07.755738  3207 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:07.756138  3207 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:07.824241  3207 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.188:36455
I20250628 07:53:07.824328  3284 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.188:36455 every 8 connection(s)
I20250628 07:53:07.826737  3207 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
I20250628 07:53:07.831393  3285 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:07.833886   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3207
I20250628 07:53:07.834553   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal/instance
I20250628 07:53:07.846972  3285 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:07.873812  3285 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:07.883790  3285 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } has no permanent_uuid. Determining permanent_uuid...
I20250628 07:53:07.898801  3285 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Bootstrap starting.
I20250628 07:53:07.905354  3285 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:07.906999  3285 log.cc:826] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Log is configured to *not* fsync() on all Append() calls
I20250628 07:53:07.911484  3285 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: No bootstrap required, opened a new log
I20250628 07:53:07.928181  3285 raft_consensus.cc:357] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:07.928755  3285 raft_consensus.cc:383] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:07.928984  3285 raft_consensus.cc:738] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Initialized, Role: FOLLOWER
I20250628 07:53:07.929666  3285 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:07.931816  3293 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:07.932345  3293 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:07.933211  3285 sys_catalog.cc:564] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:53:07.933351  3132 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 6
I20250628 07:53:07.956243  3132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Bootstrap starting.
I20250628 07:53:07.968869  3132 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:07.971951  3132 log.cc:826] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Log is configured to *not* fsync() on all Append() calls
W20250628 07:53:07.980396  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:07.984887  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:07.991993  3132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: No bootstrap required, opened a new log
I20250628 07:53:07.963828  3207 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: OK, instance_id { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" instance_seqno: 1751097184333109 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20250628 07:53:07.993112  3207 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:53:07.999339  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:07.999766  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.010986  3239 tablet.cc:2378] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20250628 07:53:08.014062  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.014364  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.030467  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.030786  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.036211  3132 raft_consensus.cc:357] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.037118  3132 raft_consensus.cc:383] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:08.037453  3132 raft_consensus.cc:738] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61070f3974844b798ec0e9a7c3c68f2f, State: Initialized, Role: FOLLOWER
I20250628 07:53:08.038648  3132 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.041774  3312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.042661  3312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
W20250628 07:53:08.044708  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.045138  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.047814  3132 sys_catalog.cc:564] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: configured and running, proceeding with master startup.
W20250628 07:53:08.059105  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.059393  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.073519  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.073896  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.087709  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.087990  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.102013  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.102375  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.117888  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.118374  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.132477  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.132880  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.148128  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.149595  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.164074  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.164567  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.167567  3293 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:08.168094  3293 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.170462  3293 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:08.172863  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:08.173882  3107 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 0.
W20250628 07:53:08.174149  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.174458  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.175467  3237 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d; no voters: 
I20250628 07:53:08.176481  3293 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:08.176820  3293 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:08.177186  3293 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 0 FOLLOWER]: Advancing to term 1
W20250628 07:53:08.178719  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.178984  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.141259  3064 master_runner.cc:186] Error getting master registration for 127.0.107.189:42143: OK, instance_id { permanent_uuid: "1ed16b61a11544a8968b81868339a936" instance_seqno: 1751097186050028 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20250628 07:53:08.182085  3064 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:53:08.188320  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.188685  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.191362  3293 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.195812  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f"
I20250628 07:53:08.196426  3107 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:08.204710  3107 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 1.
W20250628 07:53:08.205648  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.205969  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.206912  3293 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 1 election: Requested vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:08.209316  3237 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d; no voters: 
I20250628 07:53:08.210803  3293 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 FOLLOWER]: Leader election won for term 1
W20250628 07:53:08.219774  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.220729  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.225838  3293 raft_consensus.cc:695] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 LEADER]: Becoming Leader. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Running, Role: LEADER
I20250628 07:53:08.227255  3293 consensus_queue.cc:237] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
W20250628 07:53:08.237326  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.237764  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.243340  3308 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
I20250628 07:53:08.243683  3329 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.244200  3329 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
W20250628 07:53:08.243695  3308 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.251259  3331 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
W20250628 07:53:08.252127  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.252369  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.259407  3331 catalog_manager.cc:1486] Initializing Kudu cluster ID...
W20250628 07:53:08.266155  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.266407  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.280041  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.280294  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.280431  3107 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Refusing update from remote peer fe360a66f1a24378a030a69807008e4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:53:08.281913  3329 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:53:08.288257  3086 tablet.cc:2378] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20250628 07:53:08.310608  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.313174  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.312489  3312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.313611  3312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:08.325419  3333 mvcc.cc:204] Tried to move back new op lower bound from 7172494083177553920 to 7172494082989907968. Current Snapshot: MvccSnapshot[applied={T|T < 7172494083177553920}]
I20250628 07:53:08.334470  3312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.335079  3312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
W20250628 07:53:08.339257  3327 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20250628 07:53:08.339529  3327 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.340248  3331 catalog_manager.cc:1349] Generated new cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:08.340540  3331 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:08.343513  3329 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.344159  3329 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:08.354620  3327 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:08.357616  3327 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: success
W20250628 07:53:08.367259  3327 catalog_manager.cc:1578] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: Not found: root CA entry not found
W20250628 07:53:08.372670  3327 catalog_manager.cc:875] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.387588  3327 catalog_manager.cc:1578] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: Not found: root CA entry not found
W20250628 07:53:08.388006  3327 catalog_manager.cc:875] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.402878  3327 catalog_manager.cc:1578] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: Not found: root CA entry not found
W20250628 07:53:08.403234  3327 catalog_manager.cc:875] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.419589  3206 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } attempt: 7
W20250628 07:53:08.423859  3327 catalog_manager.cc:1578] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: Not found: root CA entry not found
W20250628 07:53:08.424122  3327 catalog_manager.cc:875] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.425700  3331 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:53:08.427520  3331 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:08.441283  3327 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: success
W20250628 07:53:08.450165  3327 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.465521  3327 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.470285  3206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Bootstrap starting.
W20250628 07:53:08.482532  3327 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.491142  3206 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:08.496263  3206 log.cc:826] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Log is configured to *not* fsync() on all Append() calls
W20250628 07:53:08.497373  3327 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
W20250628 07:53:08.512215  3327 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
I20250628 07:53:08.515822  3331 catalog_manager.cc:5955] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Generated new TSK 0
I20250628 07:53:08.516860  3331 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:08.521072  3331 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:08.522013  3206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: No bootstrap required, opened a new log
I20250628 07:53:08.528218  3327 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:53:08.538936  3331 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:08.562315   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.129:0
--local_ip_for_outbound_sockets=127.0.107.129
--webserver_interface=127.0.107.129
--webserver_port=0
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:53:08.575125  3206 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.576200  3206 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:08.576494  3206 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ed16b61a11544a8968b81868339a936, State: Initialized, Role: FOLLOWER
I20250628 07:53:08.577409  3206 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:08.590253  3340 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.591135  3340 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:08.592864  3206 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:53:08.593112  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:08.593961  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 0.
I20250628 07:53:08.594863  3180 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936"
I20250628 07:53:08.595387  3180 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:08.603606  3180 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 1.
I20250628 07:53:08.604800  3179 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Refusing update from remote peer fe360a66f1a24378a030a69807008e4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20250628 07:53:08.612203  3329 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:08.696717  3340 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.697418  3340 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:08.697009  3347 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:08.697873  3347 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:08.740338  3360 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20250628 07:53:08.780324  3360 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:08.780612  3360 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: loading cluster ID for follower catalog manager: success
I20250628 07:53:08.797374  3360 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: acquiring CA information for follower catalog manager: success
I20250628 07:53:08.803141  3360 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:53:09.041616  3362 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:09.042320  3362 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:09.044725  3362 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:09.045840  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:09.047161  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:09.049350  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:09.063625  3362 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:53:09.117200  3347 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:09.117857  3347 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:09.121980  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:09.122648  3347 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:09.123731  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:09.125165  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:09.125980  3347 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:53:09.235266  3329 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:09.236352  3329 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
W20250628 07:53:09.246819  3339 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:09.247470  3339 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:09.248420  3339 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:09.309732  3339 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:09.311342  3339 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.129
I20250628 07:53:09.377985  3339 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.0.107.129
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.129
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:09.379745  3339 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:09.381862  3339 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:09.423815  3373 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:09.743219  3378 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:09.744971  3378 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:09.748728  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:09.753202  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:09.754565  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:09.767808  3379 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:53:09.768406  3378 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:10.315472  3380 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:10.316143  3380 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:10.318782  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:10.321565  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:10.323360  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:10.329986  3380 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:10.330130  3381 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:53:10.849658  3383 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:10.851155  3382 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:10.851732  3383 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:10.855827  3382 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:10.861919  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:10.863077  3107 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1ed16b61a11544a8968b81868339a936 in term 1.
I20250628 07:53:10.864271  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f; no voters: 
I20250628 07:53:10.872277  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:10.876322  3383 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:10.877466  3383 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20250628 07:53:10.879846  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:10.881713  3382 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:10.882167  3383 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Starting leader election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:10.883057  3383 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:53:10.887529  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:10.902244  3383 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:10.903666  3181 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 61070f3974844b798ec0e9a7c3c68f2f in current term 2: Already voted for candidate 1ed16b61a11544a8968b81868339a936 in this term.
I20250628 07:53:10.906844  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:10.908748  3382 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:53:10.916841  3382 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20250628 07:53:10.919585  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f"
I20250628 07:53:10.923461  3181 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Rejecting Update request from peer fe360a66f1a24378a030a69807008e4d for earlier term 1. Current term is 2. Ops: []
I20250628 07:53:10.924084  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d"
I20250628 07:53:10.924960  3385 consensus_queue.cc:1046] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 }, Status: INVALID_TERM, Last received: 1.4, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20250628 07:53:10.925837  3385 raft_consensus.cc:3053] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 LEADER]: Stepping down as leader of term 1
I20250628 07:53:10.925516  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:10.926148  3385 raft_consensus.cc:738] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 LEADER]: Becoming Follower/Learner. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Running, Role: LEADER
I20250628 07:53:10.926761  3384 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
I20250628 07:53:10.927567  3383 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 2 election: Requested vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:10.927263  3385 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:10.929235  3385 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 1 FOLLOWER]: Advancing to term 2
I20250628 07:53:10.943094  3308 catalog_manager.cc:1604] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:53:11.345000  3385 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:11.345512  3385 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:11.347311  3385 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:11.349102  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:11.349699  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 2.
I20250628 07:53:11.356608  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:11.357029  3236 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d; no voters: 
I20250628 07:53:11.358752  3385 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 2 FOLLOWER]: Leader pre-election won for term 3
I20250628 07:53:11.359073  3385 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:11.359393  3385 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:53:11.370836  3385 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:11.389405  3107 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f"
I20250628 07:53:11.392268  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936"
I20250628 07:53:11.392843  3181 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:53:11.400556  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 3.
I20250628 07:53:11.402740  3236 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d; no voters: 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:11.403689  3386 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 FOLLOWER]: Leader election won for term 3
I20250628 07:53:11.409891  3385 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 3 election: Requested vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:11.417184  3386 raft_consensus.cc:695] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 LEADER]: Becoming Leader. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Running, Role: LEADER
I20250628 07:53:11.418076  3386 consensus_queue.cc:237] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:11.430476  3385 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.431272  3385 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:11.438083  3389 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:53:11.456324  3389 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:53:11.467500  3389 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:11.474431  3389 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:11.488576  3389 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:11.493894  3389 catalog_manager.cc:5966] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Loaded TSK: 0
I20250628 07:53:11.508131  3389 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:11.513923  3389 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:11.524456  3389 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:11.603081  3390 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:11.604801  3390 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:11.609442  3390 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:11.609745  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:11.610669  3181 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 61070f3974844b798ec0e9a7c3c68f2f in current term 3: Already voted for candidate fe360a66f1a24378a030a69807008e4d in this term.
I20250628 07:53:11.611428  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:11.614740  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:11.615473  3390 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 2 FOLLOWER]: Advancing to term 3
I20250628 07:53:11.629386  3390 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20250628 07:53:09.436049  3376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:11.745102  3374 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2323 milliseconds
I20250628 07:53:11.745597  3339 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
W20250628 07:53:09.424475  3372 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:11.751168  3339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:11.761345  3339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:11.763031  3339 hybrid_clock.cc:648] HybridClock initialized: now 1751097191762996 us; error 48 us; skew 500 ppm
I20250628 07:53:11.764211  3339 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:11.792024  3339 webserver.cc:469] Webserver started at http://127.0.107.129:46227/ using document root <none> and password file <none>
I20250628 07:53:11.793540  3339 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:11.793920  3339 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:11.794637  3339 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:11.801183  3339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data/instance:
uuid: "01832989f8e6457a8e6508afa155d558"
format_stamp: "Formatted at 2025-06-28 07:53:11 on dist-test-slave-kfnw"
I20250628 07:53:11.802752  3339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/instance:
uuid: "01832989f8e6457a8e6508afa155d558"
format_stamp: "Formatted at 2025-06-28 07:53:11 on dist-test-slave-kfnw"
I20250628 07:53:11.818507  3339 fs_manager.cc:696] Time spent creating directory manager: real 0.015s	user 0.004s	sys 0.008s
I20250628 07:53:11.831903  3395 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:11.833230  3339 fs_manager.cc:730] Time spent opening block manager: real 0.010s	user 0.005s	sys 0.005s
I20250628 07:53:11.833700  3339 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal
uuid: "01832989f8e6457a8e6508afa155d558"
format_stamp: "Formatted at 2025-06-28 07:53:11 on dist-test-slave-kfnw"
I20250628 07:53:11.834259  3339 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:11.879799  3181 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Refusing update from remote peer fe360a66f1a24378a030a69807008e4d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20250628 07:53:11.881263  3385 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20250628 07:53:11.896982  3107 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Refusing update from remote peer fe360a66f1a24378a030a69807008e4d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20250628 07:53:11.898581  3385 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20250628 07:53:11.915688  3390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.916347  3390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:11.930668  3386 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.931354  3386 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:11.935133  3397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.935792  3397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:11.942901  3339 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:11.947454  3339 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:11.947968  3339 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:11.950114  3385 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.952853  3385 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:11.954823  3397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.955608  3397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:11.961033  3339 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:53:11.978550  3390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:11.979490  3390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:11.979809  3339 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:53:11.980127  3339 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
I20250628 07:53:11.980460  3339 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:53:11.980706  3339 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:12.185552  3339 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.129:43251
I20250628 07:53:12.185657  3514 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.129:43251 every 8 connection(s)
I20250628 07:53:12.188174  3339 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/data/info.pb
I20250628 07:53:12.198168   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3339
I20250628 07:53:12.198772   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/instance
I20250628 07:53:12.218091   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.130:0
--local_ip_for_outbound_sockets=127.0.107.130
--webserver_interface=127.0.107.130
--webserver_port=0
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:53:12.252337  3515 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:12.252880  3515 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:12.254068  3517 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:12.254405  3517 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:12.254554  3515 heartbeater.cc:507] Master 127.0.107.188:36455 requested a full tablet report, sending...
I20250628 07:53:12.255559  3517 heartbeater.cc:507] Master 127.0.107.190:46841 requested a full tablet report, sending...
I20250628 07:53:12.258195  3250 ts_manager.cc:194] Registered new tserver with Master: 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:12.259742  3097 ts_manager.cc:194] Registered new tserver with Master: 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:12.261217  3250 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.129:35397
I20250628 07:53:12.266310  3516 heartbeater.cc:344] Connected to a master server at 127.0.107.189:42143
I20250628 07:53:12.266605  3516 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:12.267091  3516 heartbeater.cc:507] Master 127.0.107.189:42143 requested a full tablet report, sending...
I20250628 07:53:12.269716  3171 ts_manager.cc:194] Registered new tserver with Master: 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:12.422214  3390 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:12.422655  3390 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:12.424074  3390 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:12.424793  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:12.425132  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:12.426414  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:12.427070  3390 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:53:12.578008  3525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:12.578548  3525 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:12.579113  3525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:12.611253  3525 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:12.612188  3525 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.130
I20250628 07:53:12.647110  3525 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.0.107.130
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.130
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:12.648589  3525 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:12.650352  3525 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:12.666865  3532 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:12.765321  3515 heartbeater.cc:499] Master 127.0.107.188:36455 was elected leader, sending a full tablet report...
I20250628 07:53:13.364111  3538 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:13.364732  3538 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:13.366765  3538 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:13.367882  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:13.367913  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:13.369562  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:13.370177  3538 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:53:13.761113  3539 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:13.761536  3539 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:13.763000  3539 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:13.763890  3106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:13.764081  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:13.765587  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:13.766218  3539 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:53:12.667068  3534 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:12.667362  3531 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:13.874293  3533 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20250628 07:53:13.874452  3525 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:53:13.878507  3525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:13.880738  3525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:13.882058  3525 hybrid_clock.cc:648] HybridClock initialized: now 1751097193882018 us; error 54 us; skew 500 ppm
I20250628 07:53:13.882822  3525 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:13.889118  3525 webserver.cc:469] Webserver started at http://127.0.107.130:43777/ using document root <none> and password file <none>
I20250628 07:53:13.890137  3525 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:13.890352  3525 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:13.890851  3525 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:13.895527  3525 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data/instance:
uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
format_stamp: "Formatted at 2025-06-28 07:53:13 on dist-test-slave-kfnw"
I20250628 07:53:13.896605  3525 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/instance:
uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
format_stamp: "Formatted at 2025-06-28 07:53:13 on dist-test-slave-kfnw"
I20250628 07:53:13.897280  3543 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:13.897660  3543 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:13.899191  3543 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:13.900243  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:13.900337  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:13.901665  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:13.902223  3543 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:53:13.904783  3525 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.001s
I20250628 07:53:13.910394  3545 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:13.911381  3525 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.003s
I20250628 07:53:13.911718  3525 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal
uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
format_stamp: "Formatted at 2025-06-28 07:53:13 on dist-test-slave-kfnw"
I20250628 07:53:13.912045  3525 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:13.988019  3525 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:13.989590  3525 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:13.990021  3525 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:13.992575  3525 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:53:13.996662  3525 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:53:13.996882  3525 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:13.997139  3525 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:53:13.997298  3525 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:14.141573  3525 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.130:39723
I20250628 07:53:14.141651  3657 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.130:39723 every 8 connection(s)
I20250628 07:53:14.143926  3525 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/data/info.pb
I20250628 07:53:14.152137   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3525
I20250628 07:53:14.152796   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/instance
I20250628 07:53:14.178742   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.107.131:0
--local_ip_for_outbound_sockets=127.0.107.131
--webserver_interface=127.0.107.131
--webserver_port=0
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20250628 07:53:14.211836  3658 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:14.212376  3658 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:14.213809  3658 heartbeater.cc:507] Master 127.0.107.188:36455 requested a full tablet report, sending...
I20250628 07:53:14.216945  3250 ts_manager.cc:194] Registered new tserver with Master: 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:14.218714  3250 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.130:43211
I20250628 07:53:14.228193  3660 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:14.228562  3660 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:14.228710  3659 heartbeater.cc:344] Connected to a master server at 127.0.107.189:42143
I20250628 07:53:14.229074  3659 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:14.229142  3660 heartbeater.cc:507] Master 127.0.107.190:46841 requested a full tablet report, sending...
I20250628 07:53:14.229915  3659 heartbeater.cc:507] Master 127.0.107.189:42143 requested a full tablet report, sending...
I20250628 07:53:14.231598  3097 ts_manager.cc:194] Registered new tserver with Master: 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:14.232591  3171 ts_manager.cc:194] Registered new tserver with Master: 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:14.522223  3673 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:14.522608  3673 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
W20250628 07:53:14.523671  3668 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:14.524242  3668 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
I20250628 07:53:14.524247  3673 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:14.525233  3668 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20250628 07:53:14.525005  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:14.525408  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:14.526679  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:14.527376  3673 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:53:14.559053  3668 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:14.560003  3668 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.107.131
I20250628 07:53:14.594935  3668 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.107.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.0.107.131
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.107.131
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true

Tablet server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:14.596233  3668 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:14.597864  3668 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:14.613487  3675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:14.724596  3658 heartbeater.cc:499] Master 127.0.107.188:36455 was elected leader, sending a full tablet report...
I20250628 07:53:15.292742  3682 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:15.293282  3682 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.295078  3682 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:15.295792  3106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:15.296605  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:15.298444  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:15.299408  3682 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20250628 07:53:14.613504  3678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:14.615382  3668 server_base.cc:1048] running on GCE node
W20250628 07:53:14.613771  3676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:15.811976  3668 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:15.814749  3668 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:15.816186  3668 hybrid_clock.cc:648] HybridClock initialized: now 1751097195816124 us; error 79 us; skew 500 ppm
I20250628 07:53:15.816931  3668 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:15.823706  3668 webserver.cc:469] Webserver started at http://127.0.107.131:35979/ using document root <none> and password file <none>
I20250628 07:53:15.824656  3668 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:15.824910  3668 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:15.825392  3668 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:53:15.829787  3668 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data/instance:
uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
format_stamp: "Formatted at 2025-06-28 07:53:15 on dist-test-slave-kfnw"
I20250628 07:53:15.830986  3668 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/instance:
uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
format_stamp: "Formatted at 2025-06-28 07:53:15 on dist-test-slave-kfnw"
I20250628 07:53:15.838796  3668 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
I20250628 07:53:15.844727  3687 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:15.845929  3668 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
I20250628 07:53:15.846313  3668 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal
uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
format_stamp: "Formatted at 2025-06-28 07:53:15 on dist-test-slave-kfnw"
I20250628 07:53:15.846731  3668 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:15.886250  3692 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:15.886653  3692 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.888199  3692 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:15.888898  3106 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:15.889200  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:15.890493  3157 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:15.891162  3692 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20250628 07:53:15.898432  3668 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:15.899904  3668 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:15.900310  3668 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:15.902333  3693 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:15.902704  3693 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.903347  3668 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:53:15.904112  3693 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:15.905021  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:15.905299  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:15.905697  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 61070f3974844b798ec0e9a7c3c68f2f in term 3.
I20250628 07:53:15.907335  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f; no voters: fe360a66f1a24378a030a69807008e4d
I20250628 07:53:15.908986  3693 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Leader pre-election won for term 4
I20250628 07:53:15.909305  3693 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Starting leader election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:15.909618  3668 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:53:15.909672  3693 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:53:15.909900  3668 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:15.910215  3668 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:53:15.910470  3668 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:15.917634  3693 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.919652  3693 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 election: Requested vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:15.920368  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936"
I20250628 07:53:15.920805  3181 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:53:15.920660  3260 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d"
I20250628 07:53:15.927197  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 61070f3974844b798ec0e9a7c3c68f2f in term 4.
I20250628 07:53:15.928170  3083 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f; no voters: fe360a66f1a24378a030a69807008e4d
I20250628 07:53:15.928846  3693 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Leader election won for term 4
I20250628 07:53:15.930145  3693 raft_consensus.cc:695] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 LEADER]: Becoming Leader. State: Replica: 61070f3974844b798ec0e9a7c3c68f2f, State: Running, Role: LEADER
I20250628 07:53:15.931113  3693 consensus_queue.cc:237] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 3.5, Last appended by leader: 5, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.934876  3703 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 61070f3974844b798ec0e9a7c3c68f2f. Latest consensus state: current_term: 4 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:15.935624  3703 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:15.937521  3709 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:53:15.941836  3181 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 4 FOLLOWER]: Rejecting Update request from peer fe360a66f1a24378a030a69807008e4d for earlier term 3. Current term is 4. Ops: []
I20250628 07:53:15.943099  3695 consensus_queue.cc:1046] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 }, Status: INVALID_TERM, Last received: 3.5, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20250628 07:53:15.944770  3714 raft_consensus.cc:3053] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 LEADER]: Stepping down as leader of term 3
I20250628 07:53:15.945242  3714 raft_consensus.cc:738] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 LEADER]: Becoming Follower/Learner. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Running, Role: LEADER
I20250628 07:53:15.945994  3709 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:53:15.946439  3714 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 3.5, Last appended by leader: 5, Current term: 3, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:15.947880  3714 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 3 FOLLOWER]: Advancing to term 4
I20250628 07:53:15.948561  3709 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:15.948750  3709 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:15.952728  3709 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:15.956627  3709 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Loaded TSK: 0
I20250628 07:53:15.958261  3709 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:15.961119  3709 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:15.963492  3709 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:16.065059  3668 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.131:42211
I20250628 07:53:16.065201  3806 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.131:42211 every 8 connection(s)
I20250628 07:53:16.067714  3668 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/data/info.pb
I20250628 07:53:16.077153   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3668
I20250628 07:53:16.077709   430 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/instance
I20250628 07:53:16.118717  3807 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:16.119251  3807 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:16.120541  3807 heartbeater.cc:507] Master 127.0.107.188:36455 requested a full tablet report, sending...
I20250628 07:53:16.120883  3808 heartbeater.cc:344] Connected to a master server at 127.0.107.189:42143
I20250628 07:53:16.121186  3808 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:16.121337  3809 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:16.121655  3809 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:16.122133  3808 heartbeater.cc:507] Master 127.0.107.189:42143 requested a full tablet report, sending...
I20250628 07:53:16.122538  3809 heartbeater.cc:507] Master 127.0.107.190:46841 requested a full tablet report, sending...
I20250628 07:53:16.123860  3249 ts_manager.cc:194] Registered new tserver with Master: a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:16.125250  3171 ts_manager.cc:194] Registered new tserver with Master: a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:16.126164  3097 ts_manager.cc:194] Registered new tserver with Master: a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:16.128858  3097 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.107.131:36469
I20250628 07:53:16.133700   430 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
I20250628 07:53:16.189388  3097 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38336:
name: "default.Table_9fe83fcbeebb4e11b040d49634a59618"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.191593  3096 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38336:
name: "default.Table_104a5469114742b4b7c6e00eac8e93ac"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.200657  3095 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38336:
name: "default.Table_3e27178fec5347c9a4af581281fd709d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.201448  3094 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38336:
name: "default.Table_df57d88234e749dbb8a349ec4eb097f1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:53:16.206633  3097 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_9fe83fcbeebb4e11b040d49634a59618 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.222836  3096 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_104a5469114742b4b7c6e00eac8e93ac in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.225652  3095 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_3e27178fec5347c9a4af581281fd709d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.226953  3094 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_df57d88234e749dbb8a349ec4eb097f1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:53:16.241515   430 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu with pid 3064
W20250628 07:53:16.274749  3549 connection.cc:537] client connection to 127.0.107.190:46841 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:16.275583  3660 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:16.276643  3820 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.107.190:46841)
W20250628 07:53:16.296809  3517 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:16.430727  3714 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:16.431409  3714 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:16.433513  3714 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:16.434800  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 5 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936" is_pre_election: true
I20250628 07:53:16.435400  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 4.
I20250628 07:53:16.436609  3236 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d; no voters: 
I20250628 07:53:16.437434  3714 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 4 FOLLOWER]: Leader pre-election won for term 5
I20250628 07:53:16.438026  3714 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:16.438532  3714 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 4 FOLLOWER]: Advancing to term 5
W20250628 07:53:16.439581  3237 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:16.450747  3714 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:16.453159  3714 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 election: Requested vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:53:16.453718  3181 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe360a66f1a24378a030a69807008e4d" candidate_term: 5 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "1ed16b61a11544a8968b81868339a936"
W20250628 07:53:16.453682  3237 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:16.454221  3181 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 4 FOLLOWER]: Advancing to term 5
W20250628 07:53:16.456650  3237 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:16.460947  3181 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe360a66f1a24378a030a69807008e4d in term 5.
I20250628 07:53:16.462118  3236 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d; no voters: 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:16.463910  3714 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Leader election won for term 5
I20250628 07:53:16.464805  3714 raft_consensus.cc:695] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 LEADER]: Becoming Leader. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Running, Role: LEADER
I20250628 07:53:16.465715  3714 consensus_queue.cc:237] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 3.5, Last appended by leader: 5, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:16.481797  3845 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 5 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:16.487728  3845 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:16.502130  3847 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:53:16.517047  3847 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:53:16.522879  3847 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:16.523154  3847 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:16.528581  3847 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:16.532658  3847 catalog_manager.cc:5966] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Loaded TSK: 0
I20250628 07:53:16.533681  3847 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:16.537158  3847 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:16.541671  3847 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:16.567870  3250 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_3e27178fec5347c9a4af581281fd709d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.568405  3248 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_104a5469114742b4b7c6e00eac8e93ac"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.569494  3247 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_9fe83fcbeebb4e11b040d49634a59618"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:16.571602  3249 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_df57d88234e749dbb8a349ec4eb097f1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:53:16.577843  3248 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_104a5469114742b4b7c6e00eac8e93ac in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.581269  3250 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_3e27178fec5347c9a4af581281fd709d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.581939  3249 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_df57d88234e749dbb8a349ec4eb097f1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20250628 07:53:16.584170  3247 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_9fe83fcbeebb4e11b040d49634a59618 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20250628 07:53:16.632402  3809 heartbeater.cc:499] Master 127.0.107.190:46841 was elected leader, sending a full tablet report...
W20250628 07:53:16.637719  3809 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:16.772043  3250 catalog_manager.cc:2149] created HMS catalog entry for table  [id=baf2e85df61f4e19b493ede819300fdb]
I20250628 07:53:16.780246  3181 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Refusing update from remote peer fe360a66f1a24378a030a69807008e4d: Log matching property violated. Preceding OpId in replica: term: 3 index: 5. Preceding OpId from leader: term: 5 index: 7. (index mismatch)
I20250628 07:53:16.781770  3845 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20250628 07:53:16.783888  3237 consensus_peers.cc:489] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d -> Peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Couldn't send request to peer 61070f3974844b798ec0e9a7c3c68f2f. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20250628 07:53:16.790709  3851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe360a66f1a24378a030a69807008e4d. Latest consensus state: current_term: 5 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:16.791409  3851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:16.809779  3849 mvcc.cc:204] Tried to move back new op lower bound from 7172494117997318144 to 7172494116736188416. Current Snapshot: MvccSnapshot[applied={T|T < 7172494117997318144}]
I20250628 07:53:16.811096  3852 mvcc.cc:204] Tried to move back new op lower bound from 7172494117997318144 to 7172494116736188416. Current Snapshot: MvccSnapshot[applied={T|T < 7172494117997318144}]
I20250628 07:53:16.819543  3714 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:16.820423  3714 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:16.825495  3851 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "fe360a66f1a24378a030a69807008e4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:16.826175  3851 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:16.831192  3250 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45814:
table { table_name: "default.Table_3e27178fec5347c9a4af581281fd709d" } new_table_name: "default.Table_d9d428eecaeb4f1bb6e7e38a01d3b352" modify_external_catalogs: true
I20250628 07:53:16.892728  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 044e6d34b4584986bdebc4197af39e63 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:16.894888  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 044e6d34b4584986bdebc4197af39e63. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:16.916249   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:46841
--webserver_interface=127.0.107.190
--webserver_port=38719
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--hive_metastore_uris=thrift://127.0.0.1:34709
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:53:16.921173  3742 tablet_service.cc:1468] Processing CreateTablet for tablet 044e6d34b4584986bdebc4197af39e63 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:16.923310  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 044e6d34b4584986bdebc4197af39e63. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:16.973851  3862 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:16.975520  3450 tablet_service.cc:1468] Processing CreateTablet for tablet 044e6d34b4584986bdebc4197af39e63 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:16.977586  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 044e6d34b4584986bdebc4197af39e63. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:16.992022  3862 tablet_bootstrap.cc:654] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.000404  3862 log.cc:826] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Log is configured to *not* fsync() on all Append() calls
I20250628 07:53:17.007731  3864 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.017455  3862 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.017964  3862 ts_tablet_manager.cc:1397] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.045s	user 0.014s	sys 0.003s
I20250628 07:53:17.025282  3864 tablet_bootstrap.cc:654] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.026664  3866 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.031263  3246 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_802e2a3d96344b1294c73fbd013d9f26"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
07:53:17.031 [HMS - ERROR - pool-6-thread-1] (RetryingHMSHandler.java:201) AlreadyExistsException(message:Table hive.default.table_9fe83fcbeebb4e11b040d49634a59618 already exists)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1839)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1786)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_with_environment_context(HiveMetaStore.java:2035)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
	at com.sun.proxy.$Proxy25.create_table_with_environment_context(Unknown Source)
	at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$create_table_with_environment_context.getResult(ThriftHiveMetastore.java:14800)
	at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$create_table_with_environment_context.getResult(ThriftHiveMetastore.java:14784)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:104)
	at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

I20250628 07:53:17.035008  3864 log.cc:826] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Log is configured to *not* fsync() on all Append() calls
W20250628 07:53:17.035949  3247 catalog_manager.cc:2145] Already present: failed to create HMS catalog entry for table  [id=9c01aaf1124c49a0b1cf79b3e2350a64]: failed to create Hive MetaStore table: TException - service has thrown: AlreadyExistsException(message=Table hive.default.table_9fe83fcbeebb4e11b040d49634a59618 already exists)
I20250628 07:53:17.041420  3247 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_b76be2ebb8f54a57b15af30be3c83d29"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:17.043605  3866 tablet_bootstrap.cc:654] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.047881  3866 log.cc:826] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Log is configured to *not* fsync() on all Append() calls
I20250628 07:53:17.053051  3864 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.053787  3864 ts_tablet_manager.cc:1397] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.047s	user 0.016s	sys 0.000s
I20250628 07:53:17.054008  3245 master_service.cc:948] ReplaceTablet: received request to replace tablet 044e6d34b4584986bdebc4197af39e63 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.061616  3244 master_service.cc:948] ReplaceTablet: received request to replace tablet 044e6d34b4584986bdebc4197af39e63 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.075192  3245 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:17.077200  3593 tablet_service.cc:1515] Processing DeleteTablet for tablet 044e6d34b4584986bdebc4197af39e63 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.072836  3862 raft_consensus.cc:357] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.081950  3862 raft_consensus.cc:383] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.082296  3862 raft_consensus.cc:738] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.083039  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 044e6d34b4584986bdebc4197af39e63 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.086632  3862 consensus_queue.cc:260] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.092950  3245 catalog_manager.cc:6652] ReplaceTablet: tablet 044e6d34b4584986bdebc4197af39e63 deleted and replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9
I20250628 07:53:17.093164  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet 044e6d34b4584986bdebc4197af39e63 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9) from {username='slave'} at 127.0.0.1:36930
W20250628 07:53:17.105307  3237 catalog_manager.cc:4908] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): delete failed for tablet 044e6d34b4584986bdebc4197af39e63 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 044e6d34b4584986bdebc4197af39e63 already in progress: creating tablet
I20250628 07:53:17.116542  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.119169  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d0bdf781c624c34b5539c4ed11357f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.121973  3742 tablet_service.cc:1468] Processing CreateTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.122334  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.123361  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d0bdf781c624c34b5539c4ed11357f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.123941  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d0bdf781c624c34b5539c4ed11357f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.125510  3862 ts_tablet_manager.cc:1428] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.107s	user 0.038s	sys 0.000s
I20250628 07:53:17.126183  3658 heartbeater.cc:499] Master 127.0.107.188:36455 was elected leader, sending a full tablet report...
I20250628 07:53:17.131050  3807 heartbeater.cc:499] Master 127.0.107.188:36455 was elected leader, sending a full tablet report...
W20250628 07:53:17.145531  3234 catalog_manager.cc:4908] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): delete failed for tablet 044e6d34b4584986bdebc4197af39e63 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 044e6d34b4584986bdebc4197af39e63 already in progress: creating tablet
I20250628 07:53:17.147481  3866 tablet_bootstrap.cc:492] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.148048  3866 ts_tablet_manager.cc:1397] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.122s	user 0.012s	sys 0.014s
W20250628 07:53:17.148412  3237 catalog_manager.cc:4908] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): delete failed for tablet 044e6d34b4584986bdebc4197af39e63 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 044e6d34b4584986bdebc4197af39e63 already in progress: creating tablet
I20250628 07:53:17.120873  3864 raft_consensus.cc:357] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.150713  3864 raft_consensus.cc:383] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.151103  3864 raft_consensus.cc:738] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.151993  3864 consensus_queue.cc:260] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.185511  3862 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:17.191316  3862 tablet_bootstrap.cc:654] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.193269  3248 catalog_manager.cc:2149] created HMS catalog entry for table  [id=ae56e06624ea49a4a34d86e10d48ca24]
I20250628 07:53:17.205701  3864 ts_tablet_manager.cc:1428] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.152s	user 0.023s	sys 0.018s
I20250628 07:53:17.212859  3864 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.216034  3862 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.216429  3862 ts_tablet_manager.cc:1397] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.031s	user 0.008s	sys 0.004s
I20250628 07:53:17.218218  3864 tablet_bootstrap.cc:654] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.225520  3248 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45814:
table { table_name: "default.Table_104a5469114742b4b7c6e00eac8e93ac" } new_table_name: "default.Table_27ce0e6e2f5240a6b3ce43824e882ef0" modify_external_catalogs: true
I20250628 07:53:17.232020  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 044e6d34b4584986bdebc4197af39e63 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.232959  3870 tablet_replica.cc:331] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:17.233816  3870 raft_consensus.cc:2241] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.234371  3870 raft_consensus.cc:2270] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.218921  3862 raft_consensus.cc:357] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.235800  3862 raft_consensus.cc:383] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.236076  3862 raft_consensus.cc:738] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.236997  3870 ts_tablet_manager.cc:1905] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:53:17.231686  3665 tablet.cc:2378] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:53:17.235949  3866 raft_consensus.cc:357] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.244652  3866 raft_consensus.cc:383] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.244962  3866 raft_consensus.cc:738] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.245457  3862 consensus_queue.cc:260] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.246040  3866 consensus_queue.cc:260] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.247849  3862 ts_tablet_manager.cc:1428] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.031s	user 0.001s	sys 0.004s
I20250628 07:53:17.263345  3515 heartbeater.cc:499] Master 127.0.107.188:36455 was elected leader, sending a full tablet report...
I20250628 07:53:17.265152  3866 ts_tablet_manager.cc:1428] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.117s	user 0.035s	sys 0.002s
I20250628 07:53:17.266019  3866 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.272536  3864 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.275887  3871 raft_consensus.cc:491] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.276468  3871 raft_consensus.cc:513] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.286749  3243 master_service.cc:948] ReplaceTablet: received request to replace tablet 4d0bdf781c624c34b5539c4ed11357f9 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.288348  3866 tablet_bootstrap.cc:654] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.300678  3870 ts_tablet_manager.cc:1918] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.301182  3242 master_service.cc:948] ReplaceTablet: received request to replace tablet 4d0bdf781c624c34b5539c4ed11357f9 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.308308  3870 log.cc:1199] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/044e6d34b4584986bdebc4197af39e63
I20250628 07:53:17.283272  3871 leader_election.cc:290] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.273003  3864 ts_tablet_manager.cc:1397] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.060s	user 0.007s	sys 0.002s
I20250628 07:53:17.310722  3870 ts_tablet_manager.cc:1939] T 044e6d34b4584986bdebc4197af39e63 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:17.321235  3449 tablet_service.cc:1468] Processing CreateTablet for tablet e173e8f92f944d8d8774694ef27989d0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.322427  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e173e8f92f944d8d8774694ef27989d0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.323851  3593 tablet_service.cc:1468] Processing CreateTablet for tablet e173e8f92f944d8d8774694ef27989d0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.324909  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e173e8f92f944d8d8774694ef27989d0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.326427  3866 tablet_bootstrap.cc:492] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.327181  3866 ts_tablet_manager.cc:1397] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.061s	user 0.008s	sys 0.004s
I20250628 07:53:17.329859  3866 raft_consensus.cc:357] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.330632  3866 raft_consensus.cc:383] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.330914  3866 raft_consensus.cc:738] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.334280  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet 044e6d34b4584986bdebc4197af39e63 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d0bdf781c624c34b5539c4ed11357f9) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.335206  3872 tablet_replica.cc:331] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:17.336028  3872 raft_consensus.cc:2241] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.336598  3872 raft_consensus.cc:2270] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.339104  3249 catalog_manager.cc:2149] created HMS catalog entry for table  [id=342f6d03d8f840d79fefc55cc112166a]
I20250628 07:53:17.339593  3872 ts_tablet_manager.cc:1905] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.331496  3866 consensus_queue.cc:260] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.345794  3243 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet 4d0bdf781c624c34b5539c4ed11357f9
I20250628 07:53:17.348112  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 with delete_type TABLET_DATA_DELETED (replaced by tablet c7e4aa4b0a4c4fa28e5d489183f5794e) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.349352  3243 catalog_manager.cc:6652] ReplaceTablet: tablet 4d0bdf781c624c34b5539c4ed11357f9 deleted and replaced by tablet c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:17.349804  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 with delete_type TABLET_DATA_DELETED (replaced by tablet c7e4aa4b0a4c4fa28e5d489183f5794e) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.320501  3864 raft_consensus.cc:357] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.328590  3742 tablet_service.cc:1468] Processing CreateTablet for tablet e173e8f92f944d8d8774694ef27989d0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.351408  3866 ts_tablet_manager.cc:1428] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.024s	user 0.003s	sys 0.003s
I20250628 07:53:17.351550  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e173e8f92f944d8d8774694ef27989d0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.351485  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 with delete_type TABLET_DATA_DELETED (replaced by tablet c7e4aa4b0a4c4fa28e5d489183f5794e) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.352418  3869 tablet_replica.cc:331] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:17.353262  3869 raft_consensus.cc:2241] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.353816  3869 raft_consensus.cc:2270] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.361200  3237 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet 044e6d34b4584986bdebc4197af39e63 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:17.356910  3869 ts_tablet_manager.cc:1905] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:53:17.362275  3237 catalog_manager.cc:4908] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): delete failed for tablet 4d0bdf781c624c34b5539c4ed11357f9 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 4d0bdf781c624c34b5539c4ed11357f9 already in progress: creating tablet
I20250628 07:53:17.374492  3864 raft_consensus.cc:383] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.374948  3864 raft_consensus.cc:738] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.375923  3864 consensus_queue.cc:260] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.377544  3866 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.383076  3243 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45814:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
I20250628 07:53:17.383924  3243 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45814:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
I20250628 07:53:17.386502  3872 ts_tablet_manager.cc:1918] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.388382  3864 ts_tablet_manager.cc:1428] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.078s	user 0.005s	sys 0.001s
I20250628 07:53:17.395356  3864 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.388669  3866 tablet_bootstrap.cc:654] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.398356  3872 log.cc:1199] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/044e6d34b4584986bdebc4197af39e63
I20250628 07:53:17.399430  3872 ts_tablet_manager.cc:1939] T 044e6d34b4584986bdebc4197af39e63 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:17.402370  3864 tablet_bootstrap.cc:654] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.402947  3872 tablet_replica.cc:331] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:17.403080  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet 044e6d34b4584986bdebc4197af39e63 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:17.409049  3872 raft_consensus.cc:2241] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.410426  3872 raft_consensus.cc:2270] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.410921  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 4d0bdf781c624c34b5539c4ed11357f9 with delete_type TABLET_DATA_DELETED (replaced by tablet c7e4aa4b0a4c4fa28e5d489183f5794e) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.413915  3872 ts_tablet_manager.cc:1905] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.418725  3870 tablet_replica.cc:331] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:17.419389  3870 raft_consensus.cc:2241] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.419822  3870 raft_consensus.cc:2270] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.420392  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4d0bdf781c624c34b5539c4ed11357f9" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:17.421885  3870 ts_tablet_manager.cc:1905] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:53:17.422596  3549 leader_election.cc:343] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:53:17.424163  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4d0bdf781c624c34b5539c4ed11357f9" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:17.426672  3249 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_14ae0d9fd13c4744ba376a84483ed2d7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20250628 07:53:17.428110  3549 leader_election.cc:343] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Illegal state: Tablet not RUNNING: STOPPED
I20250628 07:53:17.428541  3549 leader_election.cc:304] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
W20250628 07:53:17.429220  3549 raft_consensus.cc:2684] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: unable to run election callback: Service unavailable: Thread pool token was shut down
I20250628 07:53:17.430006  3862 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:17.434900  3866 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.435369  3866 ts_tablet_manager.cc:1397] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.058s	user 0.013s	sys 0.001s
I20250628 07:53:17.436213  3862 tablet_bootstrap.cc:654] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.437659  3866 raft_consensus.cc:357] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.439092  3866 raft_consensus.cc:383] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.445209  3866 raft_consensus.cc:738] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.447604  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 4858162307a14ec0b8804aa9e089c029 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.448048  3742 tablet_service.cc:1468] Processing CreateTablet for tablet 4858162307a14ec0b8804aa9e089c029 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
F20250628 07:53:17.448158  3307 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_3e27178fec5347c9a4af581281fd709d, message={"server":"","servicePrincipal":"","db":"default","table":"table_3e27178fec5347c9a4af581281fd709d","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_3e27178fec5347c9a4af581281fd709d\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_3e27178fec5347c9a4af581281fd709d\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_3e27178fec5347c9a4af581281fd709d\",\"kudu.table_id\":\"baf2e85df61f4e19b493ede819300fdb\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_9fe83fcbeebb4e11b040d49634a59618, message={"server":"","servicePrincipal":"","db":"default","table":"table_9fe83fcbeebb4e11b040d49634a59618","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_9fe83fcbeebb4e11b040d49634a59618\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_9fe83fcbeebb4e11b040d49634a59618\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_9fe83fcbeebb4e11b040d49634a59618\",\"kudu.table_id\":\"8d922997ba434783b26f9d52c4d65f65\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive);
I20250628 07:53:17.449165  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858162307a14ec0b8804aa9e089c029. 1 dirs total, 0 dirs full, 0 dirs failed
*** Check failure stack trace: ***
I20250628 07:53:17.449339  3866 consensus_queue.cc:260] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.452874  3866 ts_tablet_manager.cc:1428] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.017s	user 0.004s	sys 0.000s
I20250628 07:53:17.459736  3864 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.460203  3864 ts_tablet_manager.cc:1397] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.065s	user 0.018s	sys 0.014s
I20250628 07:53:17.460779  3870 ts_tablet_manager.cc:1918] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.461139  3870 log.cc:1199] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/4d0bdf781c624c34b5539c4ed11357f9
I20250628 07:53:17.461853  3870 ts_tablet_manager.cc:1939] T 4d0bdf781c624c34b5539c4ed11357f9 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:17.462810  3864 raft_consensus.cc:357] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.463585  3864 raft_consensus.cc:383] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.463912  3864 raft_consensus.cc:738] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.464255  3237 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet 4d0bdf781c624c34b5539c4ed11357f9 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:17.465162  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858162307a14ec0b8804aa9e089c029. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.467473  3450 tablet_service.cc:1468] Processing CreateTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.468466  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c7e4aa4b0a4c4fa28e5d489183f5794e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.468922  3592 tablet_service.cc:1468] Processing CreateTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.470069  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c7e4aa4b0a4c4fa28e5d489183f5794e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.473841  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 4858162307a14ec0b8804aa9e089c029 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.474879  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858162307a14ec0b8804aa9e089c029. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.464828  3864 consensus_queue.cc:260] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.479061  3862 tablet_bootstrap.cc:492] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.480798  3741 tablet_service.cc:1468] Processing CreateTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.482120  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c7e4aa4b0a4c4fa28e5d489183f5794e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.489295  3864 ts_tablet_manager.cc:1428] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.029s	user 0.005s	sys 0.000s
I20250628 07:53:17.489809  3862 ts_tablet_manager.cc:1397] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.060s	user 0.002s	sys 0.010s
W20250628 07:53:17.493626  3518 tablet.cc:2378] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:53:17.488258  3869 ts_tablet_manager.cc:1918] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.505416  3869 log.cc:1199] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/4d0bdf781c624c34b5539c4ed11357f9
I20250628 07:53:17.508486  3872 ts_tablet_manager.cc:1918] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.492475  3862 raft_consensus.cc:357] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.509227  3866 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.509467  3862 raft_consensus.cc:383] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.509665  3869 ts_tablet_manager.cc:1939] T 4d0bdf781c624c34b5539c4ed11357f9 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:17.509779  3862 raft_consensus.cc:738] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.510624  3862 consensus_queue.cc:260] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.512727  3862 ts_tablet_manager.cc:1428] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.023s	user 0.002s	sys 0.001s
I20250628 07:53:17.512781  3234 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 4d0bdf781c624c34b5539c4ed11357f9 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:17.515570  3866 tablet_bootstrap.cc:654] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.517287  3872 log.cc:1199] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/4d0bdf781c624c34b5539c4ed11357f9
I20250628 07:53:17.518011  3872 ts_tablet_manager.cc:1939] T 4d0bdf781c624c34b5539c4ed11357f9 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:17.523399  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet 4d0bdf781c624c34b5539c4ed11357f9 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
    @     0x7f8ca6c2b950  google::LogMessage::Flush() at ??:0
I20250628 07:53:17.533803  3862 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:17.537751  3866 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.538180  3866 ts_tablet_manager.cc:1397] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.029s	user 0.012s	sys 0.001s
I20250628 07:53:17.541301  3864 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.540863  3866 raft_consensus.cc:357] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.541606  3866 raft_consensus.cc:383] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.541870  3866 raft_consensus.cc:738] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.542503  3866 consensus_queue.cc:260] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.543085  3862 tablet_bootstrap.cc:654] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.546933  3866 ts_tablet_manager.cc:1428] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.001s
I20250628 07:53:17.547843  3864 tablet_bootstrap.cc:654] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.555728  3242 master_service.cc:948] ReplaceTablet: received request to replace tablet c7e4aa4b0a4c4fa28e5d489183f5794e from {username='slave'} at 127.0.0.1:45814
    @     0x7f8ca6c2b6bf  google::LogMessage::~LogMessage() at ??:0
I20250628 07:53:17.562637  3245 master_service.cc:948] ReplaceTablet: received request to replace tablet c7e4aa4b0a4c4fa28e5d489183f5794e from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.572007  3866 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.580600  3866 tablet_bootstrap.cc:654] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.583137  3875 raft_consensus.cc:491] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.583722  3875 raft_consensus.cc:513] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.586472  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.587359  3872 tablet_replica.cc:331] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:17.587438  3878 raft_consensus.cc:491] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.588092  3872 raft_consensus.cc:2241] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.588582  3872 raft_consensus.cc:2270] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.588114  3878 raft_consensus.cc:513] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.589784  3871 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.590217  3871 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:17.590790  3862 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.591204  3862 ts_tablet_manager.cc:1397] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.058s	user 0.000s	sys 0.013s
I20250628 07:53:17.591179  3875 leader_election.cc:290] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:17.591979  3871 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:17.594959  3862 raft_consensus.cc:357] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.595849  3862 raft_consensus.cc:383] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.596247  3862 raft_consensus.cc:738] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
W20250628 07:53:17.596736  3812 tablet.cc:2378] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:53:17.597275  3862 consensus_queue.cc:260] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
W20250628 07:53:17.598264  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:17.600164  3872 ts_tablet_manager.cc:1905] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.600787  3862 ts_tablet_manager.cc:1428] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.009s	user 0.008s	sys 0.000s
I20250628 07:53:17.601923  3862 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
    @     0x7f8cb1b0863c  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20250628 07:53:17.612062  3878 leader_election.cc:290] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:17.612859  3862 tablet_bootstrap.cc:654] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
W20250628 07:53:17.624305  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:17.624718  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:17.626363  3871 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:17.628404  3862 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.628870  3862 ts_tablet_manager.cc:1397] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.027s	user 0.016s	sys 0.001s
I20250628 07:53:17.631636  3862 raft_consensus.cc:357] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.632306  3862 raft_consensus.cc:383] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.632597  3862 raft_consensus.cc:738] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.633288  3862 consensus_queue.cc:260] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
    @     0x7f8cb1b07dfc  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
I20250628 07:53:17.636128  3862 ts_tablet_manager.cc:1428] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.007s	user 0.002s	sys 0.003s
I20250628 07:53:17.644752  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:17.645473  3470 raft_consensus.cc:2466] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 0.
I20250628 07:53:17.645450  3245 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45814:
name: "default.Table_76b9e7d39c4a477ea1ec1a67f687c40a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:17.647030  3691 leader_election.cc:304] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:17.648080  3875 raft_consensus.cc:2802] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:17.648424  3875 raft_consensus.cc:491] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:17.648728  3875 raft_consensus.cc:3058] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.651878  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.652542  3869 tablet_replica.cc:331] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:17.653281  3869 raft_consensus.cc:2241] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.653728  3869 raft_consensus.cc:2270] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.655521  3875 raft_consensus.cc:513] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.655965  3864 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.656358  3864 ts_tablet_manager.cc:1397] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.115s	user 0.013s	sys 0.008s
I20250628 07:53:17.656738  3869 ts_tablet_manager.cc:1905] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.657934  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:17.658478  3470 raft_consensus.cc:3058] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.658772  3864 raft_consensus.cc:357] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.659408  3864 raft_consensus.cc:383] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.659688  3864 raft_consensus.cc:738] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.660295  3864 consensus_queue.cc:260] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.664767  3470 raft_consensus.cc:2466] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 1.
I20250628 07:53:17.665686  3691 leader_election.cc:304] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:17.666755  3875 leader_election.cc:290] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:17.666966  3242 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:17.669831  3242 catalog_manager.cc:6652] ReplaceTablet: tablet c7e4aa4b0a4c4fa28e5d489183f5794e deleted and replaced by tablet 31351ea090b9459ab409eb4119024431
I20250628 07:53:17.670702  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:17.671486  3613 raft_consensus.cc:2466] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 0.
I20250628 07:53:17.671649  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.672741  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.677743  3875 raft_consensus.cc:2802] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:17.678206  3875 raft_consensus.cc:695] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 LEADER]: Becoming Leader. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Running, Role: LEADER
I20250628 07:53:17.686146  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:17.686686  3613 raft_consensus.cc:3058] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.693286  3613 raft_consensus.cc:2466] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 1.
I20250628 07:53:17.697458  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 31351ea090b9459ab409eb4119024431 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.697371  3875 consensus_queue.cc:237] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.697990  3592 tablet_service.cc:1468] Processing CreateTablet for tablet 31351ea090b9459ab409eb4119024431 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.698861  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 31351ea090b9459ab409eb4119024431. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.698969  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 31351ea090b9459ab409eb4119024431. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.704032  3866 tablet_bootstrap.cc:492] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.704470  3866 ts_tablet_manager.cc:1397] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.157s	user 0.007s	sys 0.008s
I20250628 07:53:17.711508  3872 ts_tablet_manager.cc:1918] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.711459  3871 raft_consensus.cc:491] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.711854  3872 log.cc:1199] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:17.711890  3871 raft_consensus.cc:513] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.712466  3872 ts_tablet_manager.cc:1939] T c7e4aa4b0a4c4fa28e5d489183f5794e P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:17.713706  3871 leader_election.cc:290] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.713749  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:17.715668  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4858162307a14ec0b8804aa9e089c029" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:17.717250  3762 raft_consensus.cc:2466] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:17.718528  3549 leader_election.cc:304] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:17.719457  3871 raft_consensus.cc:2802] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:17.719854  3871 raft_consensus.cc:491] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:17.720240  3871 raft_consensus.cc:3058] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.723850  3869 ts_tablet_manager.cc:1918] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.724391  3869 log.cc:1199] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:17.726213  3869 ts_tablet_manager.cc:1939] T c7e4aa4b0a4c4fa28e5d489183f5794e P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:17.726613  3864 ts_tablet_manager.cc:1428] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.070s	user 0.018s	sys 0.039s
I20250628 07:53:17.728103  3864 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.727927  3871 raft_consensus.cc:513] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
    @     0x7f8cb1b0be72  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
I20250628 07:53:17.729530  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4858162307a14ec0b8804aa9e089c029" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:17.730685  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e173e8f92f944d8d8774694ef27989d0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
W20250628 07:53:17.730837  3549 leader_election.cc:343] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:17.731400  3613 raft_consensus.cc:2391] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 01832989f8e6457a8e6508afa155d558 in current term 1: Already voted for candidate a2ddfadc2e7748c0a5505a4c516406b9 in this term.
I20250628 07:53:17.732908  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4858162307a14ec0b8804aa9e089c029" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:17.734035  3864 tablet_bootstrap.cc:654] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
W20250628 07:53:17.734270  3549 leader_election.cc:343] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:17.736946  3234 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet c7e4aa4b0a4c4fa28e5d489183f5794e (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:17.737051  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4858162307a14ec0b8804aa9e089c029" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:17.737552  3762 raft_consensus.cc:3058] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.738683  3862 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
W20250628 07:53:17.738713  3234 catalog_manager.cc:4895] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): delete failed for tablet c7e4aa4b0a4c4fa28e5d489183f5794e because the tablet was not found. No further retry: Not found: Tablet not found: c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:17.743103  3762 raft_consensus.cc:2466] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
W20250628 07:53:17.743788  3691 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.107.129:43251, user_credentials={real_user=slave}} blocked reactor thread for 78168.5us
I20250628 07:53:17.744055  3549 leader_election.cc:304] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558
I20250628 07:53:17.745718  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.746945  3862 tablet_bootstrap.cc:654] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.747486  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 31351ea090b9459ab409eb4119024431 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.748548  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 31351ea090b9459ab409eb4119024431. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.751701  3902 raft_consensus.cc:2802] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election won for term 1
W20250628 07:53:17.754329  3237 catalog_manager.cc:4908] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): delete failed for tablet c7e4aa4b0a4c4fa28e5d489183f5794e because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c7e4aa4b0a4c4fa28e5d489183f5794e already in progress: creating tablet
I20250628 07:53:17.759174  3864 tablet_bootstrap.cc:492] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.759629  3864 ts_tablet_manager.cc:1397] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.032s	user 0.007s	sys 0.007s
I20250628 07:53:17.760685  3871 leader_election.cc:290] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.762666  3864 raft_consensus.cc:357] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.763396  3864 raft_consensus.cc:383] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.763700  3864 raft_consensus.cc:738] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.764374  3864 consensus_queue.cc:260] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.766813  3862 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:17.767243  3862 ts_tablet_manager.cc:1397] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.029s	user 0.010s	sys 0.000s
    @     0x7f8cb1b0be2a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
I20250628 07:53:17.767446  3402 leader_election.cc:304] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 01832989f8e6457a8e6508afa155d558; no voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:17.768471  3878 raft_consensus.cc:2747] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:17.770109  3862 raft_consensus.cc:357] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.771083  3862 raft_consensus.cc:383] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.771414  3862 raft_consensus.cc:738] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.772138  3862 consensus_queue.cc:260] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.774581  3862 ts_tablet_manager.cc:1428] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.007s	user 0.004s	sys 0.003s
I20250628 07:53:17.777566  3864 ts_tablet_manager.cc:1428] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.018s	user 0.005s	sys 0.004s
I20250628 07:53:17.777688  3902 raft_consensus.cc:695] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Becoming Leader. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Running, Role: LEADER
I20250628 07:53:17.778512  3864 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.778618  3902 consensus_queue.cc:237] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.788074  3866 raft_consensus.cc:357] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.788808  3866 raft_consensus.cc:383] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.789088  3866 raft_consensus.cc:738] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.790503  3864 tablet_bootstrap.cc:654] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.794812  3242 master_service.cc:948] ReplaceTablet: received request to replace tablet 4858162307a14ec0b8804aa9e089c029 from {username='slave'} at 127.0.0.1:45814
    @     0x7f8cb1b0bdba  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
I20250628 07:53:17.809525  3864 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.809945  3864 ts_tablet_manager.cc:1397] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.032s	user 0.011s	sys 0.000s
I20250628 07:53:17.812537  3242 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet 4858162307a14ec0b8804aa9e089c029
I20250628 07:53:17.812459  3864 raft_consensus.cc:357] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.813107  3864 raft_consensus.cc:383] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.813385  3864 raft_consensus.cc:738] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.813386  3902 raft_consensus.cc:491] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.813818  3902 raft_consensus.cc:513] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.814018  3864 consensus_queue.cc:260] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.815690  3902 leader_election.cc:290] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.816145  3864 ts_tablet_manager.cc:1428] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20250628 07:53:17.817945  3242 catalog_manager.cc:6652] ReplaceTablet: tablet 4858162307a14ec0b8804aa9e089c029 deleted and replaced by tablet 67e3047db2274d6aa27b7f277f730437
I20250628 07:53:17.818192  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:17.818719  3762 raft_consensus.cc:2466] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:17.818723  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:17.819446  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet 4858162307a14ec0b8804aa9e089c029 with delete_type TABLET_DATA_DELETED (replaced by tablet 67e3047db2274d6aa27b7f277f730437) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.820137  3869 tablet_replica.cc:331] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:17.820832  3869 raft_consensus.cc:2241] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:17.821558  3869 pending_rounds.cc:70] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Trying to abort 1 pending ops.
I20250628 07:53:17.821823  3869 pending_rounds.cc:77] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172494122111864832 op_type: NO_OP noop_request { }
I20250628 07:53:17.822239  3869 raft_consensus.cc:2887] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:53:17.822615  3869 raft_consensus.cc:2270] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:53:17.824095  3549 leader_election.cc:343] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:17.824764  3549 leader_election.cc:304] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558
I20250628 07:53:17.826004  3902 raft_consensus.cc:2802] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:17.826304  3902 raft_consensus.cc:491] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:17.828089  3902 raft_consensus.cc:3058] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.835449  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 4858162307a14ec0b8804aa9e089c029 with delete_type TABLET_DATA_DELETED (replaced by tablet 67e3047db2274d6aa27b7f277f730437) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.835682  3902 raft_consensus.cc:513] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.836005  3869 ts_tablet_manager.cc:1905] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.836977  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet 4858162307a14ec0b8804aa9e089c029 with delete_type TABLET_DATA_DELETED (replaced by tablet 67e3047db2274d6aa27b7f277f730437) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.837787  3870 tablet_replica.cc:331] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:17.838582  3870 raft_consensus.cc:2241] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.839066  3870 raft_consensus.cc:2270] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.839238  3902 leader_election.cc:290] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.841272  3870 ts_tablet_manager.cc:1905] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.841104  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:17.843500  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:17.844041  3762 raft_consensus.cc:3058] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.846722  3866 consensus_queue.cc:260] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:17.851069  3866 ts_tablet_manager.cc:1428] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.078s	user 0.005s	sys 0.000s
W20250628 07:53:17.851671  3549 leader_election.cc:343] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:17.852342  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet c7e4aa4b0a4c4fa28e5d489183f5794e (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
W20250628 07:53:17.853070  3237 catalog_manager.cc:4895] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): delete failed for tablet c7e4aa4b0a4c4fa28e5d489183f5794e because the tablet was not found. No further retry: Not found: Tablet not found: c7e4aa4b0a4c4fa28e5d489183f5794e
W20250628 07:53:17.853715  3237 catalog_manager.cc:4908] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): delete failed for tablet 4858162307a14ec0b8804aa9e089c029 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 4858162307a14ec0b8804aa9e089c029 already in progress: creating tablet
I20250628 07:53:17.855095  3869 ts_tablet_manager.cc:1918] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:17.855489  3869 log.cc:1199] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/4858162307a14ec0b8804aa9e089c029
I20250628 07:53:17.856266  3869 ts_tablet_manager.cc:1939] T 4858162307a14ec0b8804aa9e089c029 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:17.856886  3866 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.858832  3234 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 4858162307a14ec0b8804aa9e089c029 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:17.867161  3866 tablet_bootstrap.cc:654] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
    @     0x7f8cb1b0bd82  std::__1::__function::__alloc_func<>::operator()() at ??:0
I20250628 07:53:17.870566  3762 raft_consensus.cc:2466] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:17.871735  3242 catalog_manager.cc:5582] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 reported cstate change: term changed from 0 to 1, leader changed from <none> to a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131). New cstate: current_term: 1 leader_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } health_report { overall_health: UNKNOWN } } }
I20250628 07:53:17.874972  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 67e3047db2274d6aa27b7f277f730437 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.875447  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 67e3047db2274d6aa27b7f277f730437 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.875638  3592 tablet_service.cc:1468] Processing CreateTablet for tablet 67e3047db2274d6aa27b7f277f730437 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:17.876048  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67e3047db2274d6aa27b7f277f730437. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.876478  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67e3047db2274d6aa27b7f277f730437. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.876722  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 67e3047db2274d6aa27b7f277f730437. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:17.878077  3549 leader_election.cc:304] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558
I20250628 07:53:17.879091  3902 raft_consensus.cc:2802] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:17.879537  3902 raft_consensus.cc:695] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Becoming Leader. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Running, Role: LEADER
I20250628 07:53:17.880273  3902 consensus_queue.cc:237] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.888183  3241 master_service.cc:948] ReplaceTablet: received request to replace tablet e173e8f92f944d8d8774694ef27989d0 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:17.899449  3866 tablet_bootstrap.cc:492] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:17.899979  3866 ts_tablet_manager.cc:1397] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.043s	user 0.012s	sys 0.002s
I20250628 07:53:17.902498  3866 raft_consensus.cc:357] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.903681  3864 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:17.909989  3864 tablet_bootstrap.cc:654] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.914635  3870 ts_tablet_manager.cc:1918] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.915031  3870 log.cc:1199] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/4858162307a14ec0b8804aa9e089c029
I20250628 07:53:17.915643  3870 ts_tablet_manager.cc:1939] T 4858162307a14ec0b8804aa9e089c029 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:17.918067  3866 raft_consensus.cc:383] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.918414  3866 raft_consensus.cc:738] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.919041  3866 consensus_queue.cc:260] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.927286  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 4858162307a14ec0b8804aa9e089c029 with delete_type TABLET_DATA_DELETED (replaced by tablet 67e3047db2274d6aa27b7f277f730437) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.928076  3866 ts_tablet_manager.cc:1428] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.028s	user 0.001s	sys 0.005s
I20250628 07:53:17.928222  3872 tablet_replica.cc:331] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:17.929265  3872 raft_consensus.cc:2241] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.929754  3872 raft_consensus.cc:2270] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.929488  3866 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:17.931237  3237 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet 4858162307a14ec0b8804aa9e089c029 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:17.939000  3872 ts_tablet_manager.cc:1905] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.939837  3864 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:17.940261  3864 ts_tablet_manager.cc:1397] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.037s	user 0.008s	sys 0.003s
I20250628 07:53:17.942924  3864 raft_consensus.cc:357] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.943536  3864 raft_consensus.cc:383] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:17.943784  3864 raft_consensus.cc:738] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:17.945174  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet c7e4aa4b0a4c4fa28e5d489183f5794e with delete_type TABLET_DATA_DELETED (replaced by tablet 31351ea090b9459ab409eb4119024431) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.945969  3870 tablet_replica.cc:331] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:17.946772  3870 raft_consensus.cc:2241] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.947327  3870 raft_consensus.cc:2270] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.949357  3864 consensus_queue.cc:260] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.951536  3870 ts_tablet_manager.cc:1905] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.962324  3241 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet e173e8f92f944d8d8774694ef27989d0
I20250628 07:53:17.963677  3241 catalog_manager.cc:6652] ReplaceTablet: tablet e173e8f92f944d8d8774694ef27989d0 deleted and replaced by tablet eb18d6cfd9834b8db046798e69a544dd
I20250628 07:53:17.964449  3862 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:17.969241  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet e173e8f92f944d8d8774694ef27989d0 with delete_type TABLET_DATA_DELETED (replaced by tablet eb18d6cfd9834b8db046798e69a544dd) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:17.969859  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet e173e8f92f944d8d8774694ef27989d0 with delete_type TABLET_DATA_DELETED (replaced by tablet eb18d6cfd9834b8db046798e69a544dd) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:17.970911  3878 raft_consensus.cc:491] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.971328  3878 raft_consensus.cc:513] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.972082  3864 ts_tablet_manager.cc:1428] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.032s	user 0.005s	sys 0.001s
I20250628 07:53:17.973143  3878 leader_election.cc:290] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:17.974653  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:17.975221  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "31351ea090b9459ab409eb4119024431" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:17.976012  3762 raft_consensus.cc:2391] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 01832989f8e6457a8e6508afa155d558 in current term 1: Already voted for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in this term.
I20250628 07:53:17.976213  3866 tablet_bootstrap.cc:654] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.976400  3862 tablet_bootstrap.cc:654] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:17.979832  3402 leader_election.cc:304] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 01832989f8e6457a8e6508afa155d558; no voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:17.980652  3878 raft_consensus.cc:3058] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:17.983413  3872 ts_tablet_manager.cc:1918] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:17.983866  3872 log.cc:1199] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/4858162307a14ec0b8804aa9e089c029
I20250628 07:53:17.984582  3872 ts_tablet_manager.cc:1939] T 4858162307a14ec0b8804aa9e089c029 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:17.987100  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet 4858162307a14ec0b8804aa9e089c029 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:17.988873  3872 tablet_replica.cc:331] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:17.981244  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet e173e8f92f944d8d8774694ef27989d0 with delete_type TABLET_DATA_DELETED (replaced by tablet eb18d6cfd9834b8db046798e69a544dd) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:17.989672  3869 tablet_replica.cc:331] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:17.989393  3242 catalog_manager.cc:5582] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130). New cstate: current_term: 1 leader_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } health_report { overall_health: UNKNOWN } } }
I20250628 07:53:17.990435  3869 raft_consensus.cc:2241] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:17.990901  3869 raft_consensus.cc:2270] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:17.992700  3875 raft_consensus.cc:491] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:17.993271  3869 ts_tablet_manager.cc:1905] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:17.993181  3875 raft_consensus.cc:513] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:17.994992  3875 leader_election.cc:290] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:17.998131  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
W20250628 07:53:18.000022  3688 leader_election.cc:343] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:17.996526  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
W20250628 07:53:18.007256  3691 leader_election.cc:343] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:18.007688  3691 leader_election.cc:304] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:18.008737  3875 raft_consensus.cc:2747] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:18.010267  3872 raft_consensus.cc:2241] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:18.012264  3862 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:18.012737  3862 ts_tablet_manager.cc:1397] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.048s	user 0.006s	sys 0.006s
I20250628 07:53:18.015141  3862 raft_consensus.cc:357] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.016048  3872 raft_consensus.cc:2270] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:18.016415  3862 raft_consensus.cc:383] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.019426  3449 tablet_service.cc:1468] Processing CreateTablet for tablet eb18d6cfd9834b8db046798e69a544dd (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.020525  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eb18d6cfd9834b8db046798e69a544dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:18.002521  3878 raft_consensus.cc:2747] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:18.020385  3592 tablet_service.cc:1468] Processing CreateTablet for tablet eb18d6cfd9834b8db046798e69a544dd (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.021940  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eb18d6cfd9834b8db046798e69a544dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:18.022513  3741 tablet_service.cc:1468] Processing CreateTablet for tablet eb18d6cfd9834b8db046798e69a544dd (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.023576  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet eb18d6cfd9834b8db046798e69a544dd. 1 dirs total, 0 dirs full, 0 dirs failed
    @     0x7f8cb1b0b07e  std::__1::__function::__func<>::operator()() at ??:0
    @           0x59bcd5  std::__1::__function::__value_func<>::operator()() at ??:?
I20250628 07:53:18.032681  3242 master_service.cc:948] ReplaceTablet: received request to replace tablet 31351ea090b9459ab409eb4119024431 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:18.031528  3862 raft_consensus.cc:738] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.035342  3872 ts_tablet_manager.cc:1905] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:18.036419  3862 consensus_queue.cc:260] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.044029  3869 ts_tablet_manager.cc:1918] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:18.044411  3869 log.cc:1199] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/e173e8f92f944d8d8774694ef27989d0
I20250628 07:53:18.045089  3869 ts_tablet_manager.cc:1939] T e173e8f92f944d8d8774694ef27989d0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:18.046722  3862 ts_tablet_manager.cc:1428] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.034s	user 0.005s	sys 0.000s
    @           0x59bb0a  std::__1::function<>::operator()() at ??:?
I20250628 07:53:18.061481  3864 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:18.068091  3862 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:18.070874  3866 tablet_bootstrap.cc:492] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:18.070981  3234 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet e173e8f92f944d8d8774694ef27989d0 (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:18.071295  3866 ts_tablet_manager.cc:1397] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.142s	user 0.026s	sys 0.014s
I20250628 07:53:18.073644  3866 raft_consensus.cc:357] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.074251  3866 raft_consensus.cc:383] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.074502  3866 raft_consensus.cc:738] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.075675  3862 tablet_bootstrap.cc:654] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:18.077744  3242 catalog_manager.cc:5869] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Sending DeleteTablet for 3 replicas of tablet 31351ea090b9459ab409eb4119024431
I20250628 07:53:18.079159  3242 catalog_manager.cc:6652] ReplaceTablet: tablet 31351ea090b9459ab409eb4119024431 deleted and replaced by tablet 92b31985c938493cbaca3f1868f92d06
I20250628 07:53:18.080103  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 31351ea090b9459ab409eb4119024431 with delete_type TABLET_DATA_DELETED (replaced by tablet 92b31985c938493cbaca3f1868f92d06) from {username='slave'} at 127.0.0.1:36930
I20250628 07:53:18.080924  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 31351ea090b9459ab409eb4119024431 with delete_type TABLET_DATA_DELETED (replaced by tablet 92b31985c938493cbaca3f1868f92d06) from {username='slave'} at 127.0.0.1:50024
I20250628 07:53:18.082715  3593 tablet_service.cc:1515] Processing DeleteTablet for tablet 31351ea090b9459ab409eb4119024431 with delete_type TABLET_DATA_DELETED (replaced by tablet 92b31985c938493cbaca3f1868f92d06) from {username='slave'} at 127.0.0.1:44148
I20250628 07:53:18.083406  3869 tablet_replica.cc:331] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:18.076983  3870 ts_tablet_manager.cc:1918] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:18.083837  3870 log.cc:1199] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/c7e4aa4b0a4c4fa28e5d489183f5794e
I20250628 07:53:18.084100  3869 raft_consensus.cc:2241] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:18.084461  3870 ts_tablet_manager.cc:1939] T c7e4aa4b0a4c4fa28e5d489183f5794e P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:18.084767  3869 pending_rounds.cc:70] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Trying to abort 1 pending ops.
I20250628 07:53:18.084973  3869 pending_rounds.cc:77] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172494122525663232 op_type: NO_OP noop_request { }
I20250628 07:53:18.085312  3869 raft_consensus.cc:2887] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:53:18.085664  3869 raft_consensus.cc:2270] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:18.086856  3864 tablet_bootstrap.cc:654] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:18.088248  3869 ts_tablet_manager.cc:1905] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:18.079308  3866 consensus_queue.cc:260] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.096135  3866 ts_tablet_manager.cc:1428] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.025s	user 0.006s	sys 0.001s
I20250628 07:53:18.102922  3237 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet c7e4aa4b0a4c4fa28e5d489183f5794e (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:18.109774  3866 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:18.113933  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 92b31985c938493cbaca3f1868f92d06 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.114768  3872 ts_tablet_manager.cc:1918] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:18.115263  3872 log.cc:1199] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/e173e8f92f944d8d8774694ef27989d0
I20250628 07:53:18.115303  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 92b31985c938493cbaca3f1868f92d06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:18.116000  3872 ts_tablet_manager.cc:1939] T e173e8f92f944d8d8774694ef27989d0 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:18.116063  3592 tablet_service.cc:1468] Processing CreateTablet for tablet 92b31985c938493cbaca3f1868f92d06 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.117137  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 92b31985c938493cbaca3f1868f92d06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:18.122099  3872 tablet_replica.cc:331] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:18.122195  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 92b31985c938493cbaca3f1868f92d06 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:18.122751  3872 raft_consensus.cc:2241] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:18.123217  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 92b31985c938493cbaca3f1868f92d06. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:18.123167  3872 raft_consensus.cc:2270] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:18.125579  3870 tablet_replica.cc:331] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:18.126403  3870 raft_consensus.cc:2241] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:18.126907  3872 ts_tablet_manager.cc:1905] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:18.127264  3870 pending_rounds.cc:70] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Trying to abort 1 pending ops.
I20250628 07:53:18.127610  3870 pending_rounds.cc:77] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7172494122066096128 op_type: NO_OP noop_request { }
I20250628 07:53:18.128073  3870 raft_consensus.cc:2887] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20250628 07:53:18.128446  3870 raft_consensus.cc:2270] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:18.135615  3862 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:18.135964  3862 ts_tablet_manager.cc:1397] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.068s	user 0.007s	sys 0.004s
I20250628 07:53:18.136606  3864 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:18.136981  3864 ts_tablet_manager.cc:1397] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.076s	user 0.008s	sys 0.003s
I20250628 07:53:18.139117  3864 raft_consensus.cc:357] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.141773  3862 raft_consensus.cc:357] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.142330  3862 raft_consensus.cc:383] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.142544  3862 raft_consensus.cc:738] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.143129  3862 consensus_queue.cc:260] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.147589  3862 ts_tablet_manager.cc:1428] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.011s	user 0.005s	sys 0.001s
I20250628 07:53:18.158941  3870 ts_tablet_manager.cc:1905] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:18.139744  3864 raft_consensus.cc:383] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.161520  3864 raft_consensus.cc:738] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.162189  3864 consensus_queue.cc:260] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.168836  3864 ts_tablet_manager.cc:1428] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.032s	user 0.004s	sys 0.000s
I20250628 07:53:18.170120  3866 tablet_bootstrap.cc:654] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:18.174852  3872 ts_tablet_manager.cc:1918] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:18.175302  3872 log.cc:1199] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/31351ea090b9459ab409eb4119024431
I20250628 07:53:18.177610  3872 ts_tablet_manager.cc:1939] T 31351ea090b9459ab409eb4119024431 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:18.178949  3862 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:18.172578  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet e173e8f92f944d8d8774694ef27989d0 (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:18.182085  3237 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet 31351ea090b9459ab409eb4119024431 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:18.183653  3862 tablet_bootstrap.cc:654] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:18.186654  3242 master_service.cc:948] ReplaceTablet: received request to replace tablet 31351ea090b9459ab409eb4119024431 from {username='slave'} at 127.0.0.1:45814
I20250628 07:53:18.187166  3869 ts_tablet_manager.cc:1918] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:18.187489  3869 log.cc:1199] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/31351ea090b9459ab409eb4119024431
I20250628 07:53:18.188103  3869 ts_tablet_manager.cc:1939] T 31351ea090b9459ab409eb4119024431 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:18.190387  3234 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 31351ea090b9459ab409eb4119024431 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:18.212386  3864 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:18.216970  3864 tablet_bootstrap.cc:654] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:18.229530  3866 tablet_bootstrap.cc:492] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:18.229931  3866 ts_tablet_manager.cc:1397] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.120s	user 0.007s	sys 0.002s
I20250628 07:53:18.231362  3864 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:18.231884  3864 ts_tablet_manager.cc:1397] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.020s	user 0.009s	sys 0.003s
I20250628 07:53:18.232077  3866 raft_consensus.cc:357] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.232686  3866 raft_consensus.cc:383] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.234901  3864 raft_consensus.cc:357] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.235678  3864 raft_consensus.cc:383] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.236114  3864 raft_consensus.cc:738] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.236941  3864 consensus_queue.cc:260] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
    @     0x7f8ca8e46667  kudu::Thread::SuperviseThread() at ??:0
    @           0x465e60  __tsan_thread_start_func at ??:?
I20250628 07:53:18.232916  3866 raft_consensus.cc:738] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.254837  3866 consensus_queue.cc:260] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.257624  3864 ts_tablet_manager.cc:1428] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.025s	user 0.008s	sys 0.000s
I20250628 07:53:18.260062  3862 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:18.260447  3862 ts_tablet_manager.cc:1397] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.082s	user 0.016s	sys 0.032s
I20250628 07:53:18.261823  3866 ts_tablet_manager.cc:1428] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.032s	user 0.005s	sys 0.000s
I20250628 07:53:18.262581  3866 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:18.262573  3862 raft_consensus.cc:357] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.263131  3862 raft_consensus.cc:383] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.263356  3862 raft_consensus.cc:738] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.263917  3862 consensus_queue.cc:260] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.265928  3862 ts_tablet_manager.cc:1428] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.005s	user 0.006s	sys 0.000s
I20250628 07:53:18.287042  3244 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:45814
    @     0x7f8caa4c56db  start_thread at ??:0
I20250628 07:53:18.309966  3870 ts_tablet_manager.cc:1918] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:18.310448  3870 log.cc:1199] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/e173e8f92f944d8d8774694ef27989d0
I20250628 07:53:18.315922  3866 tablet_bootstrap.cc:654] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
    @     0x7f8ca58d071f  clone at ??:0
I20250628 07:53:18.317551  3870 ts_tablet_manager.cc:1939] T e173e8f92f944d8d8774694ef27989d0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:18.321622  3870 tablet_replica.cc:331] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:18.322350  3870 raft_consensus.cc:2241] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:18.322779  3870 raft_consensus.cc:2270] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:18.324533  3870 ts_tablet_manager.cc:1905] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:18.335335  3866 tablet_bootstrap.cc:492] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:18.335680  3866 ts_tablet_manager.cc:1397] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.073s	user 0.005s	sys 0.005s
I20250628 07:53:18.337745  3866 raft_consensus.cc:357] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.338328  3866 raft_consensus.cc:383] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:18.338675  3866 raft_consensus.cc:738] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:18.339294  3866 consensus_queue.cc:260] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.343930  3866 ts_tablet_manager.cc:1428] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
I20250628 07:53:18.355302  3902 raft_consensus.cc:491] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:18.355751  3902 raft_consensus.cc:513] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.356609  3870 ts_tablet_manager.cc:1918] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:18.356972  3870 log.cc:1199] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/31351ea090b9459ab409eb4119024431
I20250628 07:53:18.357626  3870 ts_tablet_manager.cc:1939] T 31351ea090b9459ab409eb4119024431 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:18.357664  3902 leader_election.cc:290] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:18.360590  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:18.361182  3470 raft_consensus.cc:2466] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:18.362118  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:18.362701  3762 raft_consensus.cc:2466] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:18.364094  3549 leader_election.cc:304] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:18.365433  3902 raft_consensus.cc:2802] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:18.365758  3902 raft_consensus.cc:491] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:18.366080  3902 raft_consensus.cc:3058] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.373591  3902 raft_consensus.cc:513] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.376112  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:18.376664  3470 raft_consensus.cc:3058] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.383877  3470 raft_consensus.cc:2466] T 67e3047db2274d6aa27b7f277f730437 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:18.384914  3549 leader_election.cc:304] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:18.395058  3932 raft_consensus.cc:2802] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:18.396440  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "67e3047db2274d6aa27b7f277f730437" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:18.396950  3762 raft_consensus.cc:3058] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.404116  3762 raft_consensus.cc:2466] T 67e3047db2274d6aa27b7f277f730437 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:18.408272  3932 raft_consensus.cc:695] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Becoming Leader. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Running, Role: LEADER
I20250628 07:53:18.409160  3932 consensus_queue.cc:237] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.409543  3902 leader_election.cc:290] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:18.413161  3902 raft_consensus.cc:491] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:18.413590  3902 raft_consensus.cc:513] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.416184  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "92b31985c938493cbaca3f1868f92d06" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:18.416785  3470 raft_consensus.cc:2466] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:18.417559  3875 raft_consensus.cc:491] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:18.418016  3875 raft_consensus.cc:513] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.418710  3549 leader_election.cc:304] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:18.419659  3933 raft_consensus.cc:2802] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:18.419803  3875 leader_election.cc:290] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:18.420009  3933 raft_consensus.cc:491] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:18.420396  3933 raft_consensus.cc:3058] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.420635  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "92b31985c938493cbaca3f1868f92d06" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:18.421406  3762 raft_consensus.cc:2466] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:18.422358  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "eb18d6cfd9834b8db046798e69a544dd" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:18.422746  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "eb18d6cfd9834b8db046798e69a544dd" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:18.422961  3470 raft_consensus.cc:2466] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 0.
I20250628 07:53:18.423280  3613 raft_consensus.cc:2466] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 0.
I20250628 07:53:18.424471  3691 leader_election.cc:304] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:18.424907  3902 leader_election.cc:290] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:18.425870  3875 raft_consensus.cc:2802] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:18.426151  3875 raft_consensus.cc:491] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:18.426416  3875 raft_consensus.cc:3058] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.437795  3933 raft_consensus.cc:513] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.444464  3875 raft_consensus.cc:513] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.447048  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "eb18d6cfd9834b8db046798e69a544dd" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:18.447626  3470 raft_consensus.cc:3058] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.448119  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "eb18d6cfd9834b8db046798e69a544dd" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:18.448627  3613 raft_consensus.cc:3058] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.449216  3933 leader_election.cc:290] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:18.454991  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "92b31985c938493cbaca3f1868f92d06" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:18.455204  3469 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "92b31985c938493cbaca3f1868f92d06" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
I20250628 07:53:18.455519  3762 raft_consensus.cc:3058] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.455758  3469 raft_consensus.cc:3058] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:18.462502  3762 raft_consensus.cc:2466] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:18.462873  3469 raft_consensus.cc:2466] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:18.463627  3549 leader_election.cc:304] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:18.464397  3933 raft_consensus.cc:2802] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:18.465689  3933 raft_consensus.cc:695] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Becoming Leader. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Running, Role: LEADER
I20250628 07:53:18.466442  3470 raft_consensus.cc:2466] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 1.
I20250628 07:53:18.467676  3691 leader_election.cc:304] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:18.466750  3933 consensus_queue.cc:237] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:18.468308  3613 raft_consensus.cc:2466] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a2ddfadc2e7748c0a5505a4c516406b9 in term 1.
I20250628 07:53:18.475067  3875 leader_election.cc:290] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:18.475577  3875 raft_consensus.cc:2802] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:18.479275  3875 raft_consensus.cc:695] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 LEADER]: Becoming Leader. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Running, Role: LEADER
I20250628 07:53:18.480176  3875 consensus_queue.cc:237] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
W20250628 07:53:18.546571  3863 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:18.547346  3863 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:18.548141  3863 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:18.605124  3863 flags.cc:425] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20250628 07:53:18.605479  3863 flags.cc:425] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20250628 07:53:18.605865  3863 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:53:18.606251  3863 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:18.606611  3863 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:53:18.606945  3863 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:53:18.652006  3863 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
--hive_metastore_uris=thrift://127.0.0.1:34709
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.190:46841
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.107.190
--webserver_port=38719
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:18.653451  3863 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:18.655258  3863 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:18.672657  3944 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:18.688764  3691 connection.cc:537] server connection from 127.0.0.1:50024 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:18.689635  3819 connection.cc:537] client connection to 127.0.107.188:36455 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:18.691195  3690 connection.cc:537] client connection to 127.0.107.188:36455 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:18.692413  3819 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.107.188:36455)
W20250628 07:53:18.692525  3548 connection.cc:537] client connection to 127.0.107.188:36455 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:18.695456  3400 connection.cc:537] client connection to 127.0.107.188:36455 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:18.696256  3819 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.107.188:36455)
W20250628 07:53:18.698797  3819 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.107.188:36455)
W20250628 07:53:18.700724  3819 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.107.188:36455)
W20250628 07:53:18.702267  3850 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:18.694674 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.188:36455 (local address 127.0.107.130:57691)
0628 07:53:18.695451 (+   777us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:18.695500 (+    49us) client_negotiation.cc:174] Beginning negotiation
0628 07:53:18.695764 (+   264us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:53:18.700788 (+  5024us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":70}
W20250628 07:53:18.705480  3892 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:18.697217 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.188:36455 (local address 127.0.107.129:36087)
0628 07:53:18.698437 (+  1220us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:18.698502 (+    65us) client_negotiation.cc:174] Beginning negotiation
0628 07:53:18.698816 (+   314us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:53:18.704002 (+  5186us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":392}
I20250628 07:53:18.809406  3951 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:18.810102  3951 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:18.821246  3951 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:18.824359  3157 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111) [suppressed 6 similar messages]
W20250628 07:53:18.841647  3157 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
W20250628 07:53:18.847394  3158 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:18.850895  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:18.852119  3951 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
I20250628 07:53:19.299348  3958 consensus_queue.cc:1035] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:53:19.359977  3960 consensus_queue.cc:1035] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:53:19.404456  3957 consensus_queue.cc:1035] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:53:19.441295  3960 consensus_queue.cc:1035] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:19.562175  3957 consensus_queue.cc:1035] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20250628 07:53:19.620062  3974 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:19.620770  3974 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:19.651090  3958 consensus_queue.cc:1035] T 67e3047db2274d6aa27b7f277f730437 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
W20250628 07:53:19.693784  3691 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.107.129:43251, user_credentials={real_user=slave}} blocked reactor thread for 139920us
I20250628 07:53:19.763993  3974 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:19.785399  3157 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
W20250628 07:53:19.787653  3158 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:19.788268  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ed16b61a11544a8968b81868339a936; no voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:19.789759  3974 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
W20250628 07:53:18.672969  3946 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:18.674891  3943 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:20.163103  3863 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
W20250628 07:53:20.163684  3945 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1491 milliseconds
I20250628 07:53:20.171574  3863 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:20.174824  3863 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:20.176229  3863 hybrid_clock.cc:648] HybridClock initialized: now 1751097200176159 us; error 64 us; skew 500 ppm
I20250628 07:53:20.177218  3863 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:20.188448  3863 webserver.cc:469] Webserver started at http://127.0.107.190:38719/ using document root <none> and password file <none>
I20250628 07:53:20.189603  3863 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:20.189867  3863 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:20.200126  3863 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.000s
I20250628 07:53:20.230304  3987 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:20.231824  3863 fs_manager.cc:730] Time spent opening block manager: real 0.029s	user 0.010s	sys 0.007s
I20250628 07:53:20.232231  3863 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
uuid: "61070f3974844b798ec0e9a7c3c68f2f"
format_stamp: "Formatted at 2025-06-28 07:53:04 on dist-test-slave-kfnw"
I20250628 07:53:20.234990  3863 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:20.305262  3863 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:20.306697  3863 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:20.307132  3863 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:20.383460  3863 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.190:46841
I20250628 07:53:20.383544  4041 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.190:46841 every 8 connection(s)
I20250628 07:53:20.386256  3863 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-0/data/info.pb
I20250628 07:53:20.387110   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 3863
I20250628 07:53:20.397686  4042 sys_catalog.cc:263] Verifying existing consensus state
I20250628 07:53:20.402769  4042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Bootstrap starting.
I20250628 07:53:20.415992  4042 log.cc:826] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Log is configured to *not* fsync() on all Append() calls
I20250628 07:53:20.431627  4042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20250628 07:53:20.432495  4042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Bootstrap complete.
I20250628 07:53:20.453127  4042 raft_consensus.cc:357] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:20.455853  4042 raft_consensus.cc:738] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61070f3974844b798ec0e9a7c3c68f2f, State: Initialized, Role: FOLLOWER
I20250628 07:53:20.456605  4042 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 4.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:20.459326  4046 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:20.460058  4046 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:20.460819  4042 sys_catalog.cc:564] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:53:20.505568  4061 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:20.505851  4061 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: loading cluster ID for follower catalog manager: success
I20250628 07:53:20.513113  4061 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: acquiring CA information for follower catalog manager: success
I20250628 07:53:20.518574  4061 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:53:20.532439  3863 master_runner.cc:186] Error getting master registration for 127.0.107.188:36455: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111), 
I20250628 07:53:20.558821  3863 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:20.636941  4065 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:20.637470  4065 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:20.641454  4065 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:20.647015  3157 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:20.651844  4017 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 6 candidate_status { last_received { term: 5 index: 27 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f" is_pre_election: true
I20250628 07:53:20.652771  4017 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1ed16b61a11544a8968b81868339a936 in term 4.
I20250628 07:53:20.653894  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f; no voters: fe360a66f1a24378a030a69807008e4d
I20250628 07:53:20.654500  4065 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Leader pre-election won for term 6
I20250628 07:53:20.654826  4065 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Starting leader election (detected failure of leader fe360a66f1a24378a030a69807008e4d)
I20250628 07:53:20.655164  4065 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 5 FOLLOWER]: Advancing to term 6
I20250628 07:53:20.662751  4065 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:20.665040  4065 leader_election.cc:290] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 election: Requested vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:20.665444  4017 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ed16b61a11544a8968b81868339a936" candidate_term: 6 candidate_status { last_received { term: 5 index: 27 } } ignore_live_leader: false dest_uuid: "61070f3974844b798ec0e9a7c3c68f2f"
I20250628 07:53:20.665977  4017 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 4 FOLLOWER]: Advancing to term 6
W20250628 07:53:20.668749  3157 leader_election.cc:336] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:20.675220  4017 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1ed16b61a11544a8968b81868339a936 in term 6.
I20250628 07:53:20.677682  3158 leader_election.cc:304] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f; no voters: fe360a66f1a24378a030a69807008e4d
I20250628 07:53:20.682952  4065 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 6 FOLLOWER]: Leader election won for term 6
I20250628 07:53:20.692319  4065 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [term 6 LEADER]: Becoming Leader. State: Replica: 1ed16b61a11544a8968b81868339a936, State: Running, Role: LEADER
I20250628 07:53:20.693498  4065 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 26, Committed index: 26, Last appended: 5.27, Last appended by leader: 27, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:20.699795  3517 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:20.700976  4069 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1ed16b61a11544a8968b81868339a936. Latest consensus state: current_term: 6 leader_uuid: "1ed16b61a11544a8968b81868339a936" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:20.701738  4069 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:20.707134  4007 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "01832989f8e6457a8e6508afa155d558" instance_seqno: 1751097192145070) as {username='slave'} at 127.0.107.129:57647; Asking this server to re-register.
I20250628 07:53:20.708788  3517 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:20.712461  4007 ts_manager.cc:194] Registered new tserver with Master: 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:20.789839  3809 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:20.790259  3809 heartbeater.cc:499] Master 127.0.107.190:46841 was elected leader, sending a full tablet report...
I20250628 07:53:20.794130  4007 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" instance_seqno: 1751097196032695) as {username='slave'} at 127.0.107.131:58105; Asking this server to re-register.
I20250628 07:53:20.795467  3809 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:20.798137  4007 ts_manager.cc:194] Registered new tserver with Master: a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:20.828357  3660 heartbeater.cc:344] Connected to a master server at 127.0.107.190:46841
I20250628 07:53:20.832312  4007 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" instance_seqno: 1751097194108611) as {username='slave'} at 127.0.107.130:40351; Asking this server to re-register.
I20250628 07:53:20.834015  3660 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:20.837002  4007 ts_manager.cc:194] Registered new tserver with Master: 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:21.182955  4017 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Refusing update from remote peer 1ed16b61a11544a8968b81868339a936: Log matching property violated. Preceding OpId in replica: term: 4 index: 6. Preceding OpId from leader: term: 6 index: 28. (index mismatch)
I20250628 07:53:21.184594  4069 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [LEADER]: Connected to new peer: Peer: permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 28, Last known committed idx: 5, Time since last communication: 0.000s
I20250628 07:53:21.185206  4069 consensus_queue.cc:1230] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [LEADER]: Peer 61070f3974844b798ec0e9a7c3c68f2f log is divergent from this leader: its last log entry 4.6 is not in this leader's log and it has not received anything from this leader yet. Falling back to committed index 5
I20250628 07:53:21.191229  4017 pending_rounds.cc:85] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Aborting all ops after (but not including) 5
I20250628 07:53:21.191500  4017 pending_rounds.cc:107] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Aborting uncommitted NO_OP operation due to leader change: 4.6
I20250628 07:53:21.191735  4017 raft_consensus.cc:2887] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted by new leader
I20250628 07:53:21.234488  4074 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1ed16b61a11544a8968b81868339a936. Latest consensus state: current_term: 6 leader_uuid: "1ed16b61a11544a8968b81868339a936" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:21.235198  4074 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:21.239782  4077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 6 leader_uuid: "1ed16b61a11544a8968b81868339a936" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:21.240566  4077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:21.260172  4077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 6 leader_uuid: "1ed16b61a11544a8968b81868339a936" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:21.261059  4077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:21.275768  4079 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 6 leader_uuid: "1ed16b61a11544a8968b81868339a936" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:21.276480  4079 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 [sys.catalog]: This master's current role is: LEADER
W20250628 07:53:21.287753  3157 consensus_peers.cc:489] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936 -> Peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Couldn't send request to peer fe360a66f1a24378a030a69807008e4d. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20250628 07:53:21.311183  4075 mvcc.cc:204] Tried to move back new op lower bound from 7172494124247691264 to 7172494116736188416. Current Snapshot: MvccSnapshot[applied={T|T < 7172494124247691264}]
I20250628 07:53:21.354374  4071 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:53:21.360800  4071 catalog_manager.cc:671] Loaded metadata for table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]
I20250628 07:53:21.362177  4071 catalog_manager.cc:671] Loaded metadata for table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]
I20250628 07:53:21.363317  4071 catalog_manager.cc:671] Loaded metadata for table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]
I20250628 07:53:21.368669  4071 tablet_loader.cc:96] loaded metadata for tablet 5cf5b2d071654ba290785694361f54ea (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a])
I20250628 07:53:21.370541  4071 tablet_loader.cc:96] loaded metadata for tablet 92b31985c938493cbaca3f1868f92d06 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb])
I20250628 07:53:21.371444  4071 tablet_loader.cc:96] loaded metadata for tablet eb18d6cfd9834b8db046798e69a544dd (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24])
I20250628 07:53:21.372359  4071 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:53:21.375633  4071 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:21.375823  4071 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:21.379643  4071 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:21.383049  4071 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Loaded TSK: 0
I20250628 07:53:21.383956  4071 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:21.387091  4071 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:21.390187  4071 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:21.406498  3170 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:38214
I20250628 07:53:21.409507  3171 master_service.cc:948] ReplaceTablet: received request to replace tablet eb18d6cfd9834b8db046798e69a544dd from {username='slave'} at 127.0.0.1:38214
I20250628 07:53:21.405814  3167 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:38214:
table { table_name: "default.Table_104a5469114742b4b7c6e00eac8e93ac" } new_table_name: "default.Table_27ce0e6e2f5240a6b3ce43824e882ef0" modify_external_catalogs: true
I20250628 07:53:21.405942  3169 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38214:
name: "default.Table_14ae0d9fd13c4744ba376a84483ed2d7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:21.406816  3166 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38214:
name: "default.Table_b76be2ebb8f54a57b15af30be3c83d29"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:21.410735  3170 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38214:
name: "default.Table_802e2a3d96344b1294c73fbd013d9f26"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:21.408135  3165 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:38214:
table { table_name: "default.Table_3e27178fec5347c9a4af581281fd709d" } new_table_name: "default.Table_d9d428eecaeb4f1bb6e7e38a01d3b352" modify_external_catalogs: true
I20250628 07:53:21.409451  3164 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38214:
name: "default.Table_76b9e7d39c4a477ea1ec1a67f687c40a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:21.400811  3168 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:38214:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
I20250628 07:53:21.432129  3168 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:38214:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
F20250628 07:53:21.438190  3358 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_3e27178fec5347c9a4af581281fd709d, message={"server":"","servicePrincipal":"","db":"default","table":"table_3e27178fec5347c9a4af581281fd709d","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_3e27178fec5347c9a4af581281fd709d\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_3e27178fec5347c9a4af581281fd709d\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_3e27178fec5347c9a4af581281fd709d\",\"kudu.table_id\":\"baf2e85df61f4e19b493ede819300fdb\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_9fe83fcbeebb4e11b040d49634a59618, message={"server":"","servicePrincipal":"","db":"default","table":"table_9fe83fcbeebb4e11b040d49634a59618","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_9fe83fcbeebb4e11b040d49634a59618\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_9fe83fcbeebb4e11b040d49634a59618\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_9fe83fcbeebb4e11b040d49634a59618\",\"kudu.table_id\":\"8d922997ba434783b26f9d52c4d65f65\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive);
*** Check failure stack trace: ***
I20250628 07:53:21.467734  3171 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Sending DeleteTablet for 3 replicas of tablet eb18d6cfd9834b8db046798e69a544dd
I20250628 07:53:21.470105  3171 catalog_manager.cc:6652] ReplaceTablet: tablet eb18d6cfd9834b8db046798e69a544dd deleted and replaced by tablet c46915dd9de248679ebd1a9ad02db75b
    @     0x7ff55bde1950  google::LogMessage::Flush() at ??:0
I20250628 07:53:21.510524  3995 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:21.510946  3995 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:21.512351  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 5cf5b2d071654ba290785694361f54ea (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.512682  3995 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:21.513711  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cf5b2d071654ba290785694361f54ea. 1 dirs total, 0 dirs full, 0 dirs failed
W20250628 07:53:21.516070  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:21.516482  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet eb18d6cfd9834b8db046798e69a544dd with delete_type TABLET_DATA_DELETED (replaced by tablet c46915dd9de248679ebd1a9ad02db75b) from {username='slave'} at 127.0.0.1:36942
W20250628 07:53:21.516687  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:21.517016  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:21.517964  3995 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:21.542917  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet eb18d6cfd9834b8db046798e69a544dd with delete_type TABLET_DATA_DELETED (replaced by tablet c46915dd9de248679ebd1a9ad02db75b) from {username='slave'} at 127.0.0.1:44150
I20250628 07:53:21.544792  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 5cf5b2d071654ba290785694361f54ea (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.546126  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cf5b2d071654ba290785694361f54ea. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.546767  4096 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
    @     0x7ff55bde16bf  google::LogMessage::~LogMessage() at ??:0
I20250628 07:53:21.557721  4096 tablet_bootstrap.cc:654] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.573747  3591 tablet_service.cc:1468] Processing CreateTablet for tablet c46915dd9de248679ebd1a9ad02db75b (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.574855  3591 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c46915dd9de248679ebd1a9ad02db75b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.577961  3449 tablet_service.cc:1468] Processing CreateTablet for tablet c46915dd9de248679ebd1a9ad02db75b (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.578926  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c46915dd9de248679ebd1a9ad02db75b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.584537  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet eb18d6cfd9834b8db046798e69a544dd with delete_type TABLET_DATA_DELETED (replaced by tablet c46915dd9de248679ebd1a9ad02db75b) from {username='slave'} at 127.0.0.1:50030
I20250628 07:53:21.591104  4100 tablet_replica.cc:331] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:21.591871  4100 raft_consensus.cc:2241] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:21.593344  4096 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:21.593890  4096 ts_tablet_manager.cc:1397] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.047s	user 0.005s	sys 0.006s
I20250628 07:53:21.595151  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 5cf5b2d071654ba290785694361f54ea (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.595798  4100 raft_consensus.cc:2270] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.596463  4096 raft_consensus.cc:357] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.598256  4096 raft_consensus.cc:383] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:21.598593  4096 raft_consensus.cc:738] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:21.596446  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5cf5b2d071654ba290785694361f54ea. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.601985  3742 tablet_service.cc:1468] Processing CreateTablet for tablet c46915dd9de248679ebd1a9ad02db75b (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.602906  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c46915dd9de248679ebd1a9ad02db75b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.605616  4096 consensus_queue.cc:260] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.606959  4100 ts_tablet_manager.cc:1905] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.613976  4101 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:21.614585  4096 ts_tablet_manager.cc:1428] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.020s	user 0.008s	sys 0.001s
I20250628 07:53:21.620429  4101 tablet_bootstrap.cc:654] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.621263  4096 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:21.623631  3163 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:38214
I20250628 07:53:21.631528  4102 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:21.632019  4102 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.634791  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5cf5b2d071654ba290785694361f54ea" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
W20250628 07:53:21.636484  3402 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:21.638083  3516 heartbeater.cc:499] Master 127.0.107.189:42143 was elected leader, sending a full tablet report...
W20250628 07:53:21.639809  3396 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:21.640194  3396 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 01832989f8e6457a8e6508afa155d558; no voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:21.643520  4102 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:21.644428  4102 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:53:21.646584  3163 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
I20250628 07:53:21.650645  4096 tablet_bootstrap.cc:654] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.652029  4100 ts_tablet_manager.cc:1918] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:21.655311  4097 tablet_replica.cc:331] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
    @     0x7ff566cbe63c  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20250628 07:53:21.656448  4097 raft_consensus.cc:2241] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.656808  4100 log.cc:1199] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/eb18d6cfd9834b8db046798e69a544dd
I20250628 07:53:21.656970  4097 raft_consensus.cc:2270] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.657562  4100 ts_tablet_manager.cc:1939] T eb18d6cfd9834b8db046798e69a544dd P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:21.658871  4097 ts_tablet_manager.cc:1905] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.662580  3158 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet eb18d6cfd9834b8db046798e69a544dd (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:21.667160  4101 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:21.666263  3163 catalog_manager.cc:5582] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130). New cstate: current_term: 1 leader_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } }
I20250628 07:53:21.667678  4101 ts_tablet_manager.cc:1397] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.054s	user 0.008s	sys 0.000s
I20250628 07:53:21.675366  4101 raft_consensus.cc:357] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.678212  4096 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:21.678764  4096 ts_tablet_manager.cc:1397] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.058s	user 0.010s	sys 0.000s
I20250628 07:53:21.680518  4101 raft_consensus.cc:383] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:21.684285  4096 raft_consensus.cc:357] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.684990  4101 raft_consensus.cc:738] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:21.689875  3171 master_service.cc:948] ReplaceTablet: received request to replace tablet c46915dd9de248679ebd1a9ad02db75b from {username='slave'} at 127.0.0.1:38214
I20250628 07:53:21.690461  4101 consensus_queue.cc:260] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.691275  4096 raft_consensus.cc:383] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:21.691848  4096 raft_consensus.cc:738] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:21.700140  4101 ts_tablet_manager.cc:1428] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.032s	user 0.002s	sys 0.005s
I20250628 07:53:21.700915  4096 consensus_queue.cc:260] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.705672  4096 ts_tablet_manager.cc:1428] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.027s	user 0.006s	sys 0.000s
    @     0x7ff566cbddfc  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
    @     0x7ff566cc1e72  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
I20250628 07:53:21.761746  4103 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:21.775731  3659 heartbeater.cc:499] Master 127.0.107.189:42143 was elected leader, sending a full tablet report...
I20250628 07:53:21.777185  3171 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Sending DeleteTablet for 3 replicas of tablet c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:21.781844  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet c46915dd9de248679ebd1a9ad02db75b with delete_type TABLET_DATA_DELETED (replaced by tablet ec01f69277c349f491c074a00d015af0) from {username='slave'} at 127.0.0.1:44150
W20250628 07:53:21.783896  3162 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
W20250628 07:53:21.784216  3162 catalog_manager.cc:5415] Ignoring report from unknown tablet 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:21.785562  4095 tablet_replica.cc:331] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:21.786689  4095 raft_consensus.cc:2241] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.787078  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet c46915dd9de248679ebd1a9ad02db75b with delete_type TABLET_DATA_DELETED (replaced by tablet ec01f69277c349f491c074a00d015af0) from {username='slave'} at 127.0.0.1:44150
I20250628 07:53:21.787468  4095 raft_consensus.cc:2270] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.787614  3171 catalog_manager.cc:6652] ReplaceTablet: tablet c46915dd9de248679ebd1a9ad02db75b deleted and replaced by tablet ec01f69277c349f491c074a00d015af0
    @     0x7ff566cc1e2a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
I20250628 07:53:21.791821  4095 ts_tablet_manager.cc:1905] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.795614  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet c46915dd9de248679ebd1a9ad02db75b with delete_type TABLET_DATA_DELETED (replaced by tablet ec01f69277c349f491c074a00d015af0) from {username='slave'} at 127.0.0.1:36942
I20250628 07:53:21.796298  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet c46915dd9de248679ebd1a9ad02db75b with delete_type TABLET_DATA_DELETED (replaced by tablet ec01f69277c349f491c074a00d015af0) from {username='slave'} at 127.0.0.1:50030
W20250628 07:53:21.803679  3158 catalog_manager.cc:4895] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): delete failed for tablet c46915dd9de248679ebd1a9ad02db75b because the tablet was not found. No further retry: Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
    @     0x7ff566cc1dba  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
I20250628 07:53:21.837334  3163 master_service.cc:948] ReplaceTablet: received request to replace tablet 5cf5b2d071654ba290785694361f54ea from {username='slave'} at 127.0.0.1:38214
    @     0x7ff566cc1d82  std::__1::__function::__alloc_func<>::operator()() at ??:0
I20250628 07:53:21.850680  4103 tablet_bootstrap.cc:654] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.851078  4101 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:21.856925  4101 tablet_bootstrap.cc:654] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.868719  4097 ts_tablet_manager.cc:1918] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:21.871836  3593 tablet_service.cc:1468] Processing CreateTablet for tablet ec01f69277c349f491c074a00d015af0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.872884  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec01f69277c349f491c074a00d015af0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.874253  4095 ts_tablet_manager.cc:1918] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:21.874644  4095 log.cc:1199] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/eb18d6cfd9834b8db046798e69a544dd
I20250628 07:53:21.875511  4095 ts_tablet_manager.cc:1939] T eb18d6cfd9834b8db046798e69a544dd P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:21.877131  4101 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:21.877481  4101 ts_tablet_manager.cc:1397] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.027s	user 0.008s	sys 0.003s
I20250628 07:53:21.869061  4097 log.cc:1199] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/eb18d6cfd9834b8db046798e69a544dd
I20250628 07:53:21.879138  4097 ts_tablet_manager.cc:1939] T eb18d6cfd9834b8db046798e69a544dd P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:21.879884  4101 raft_consensus.cc:357] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.880290  3450 tablet_service.cc:1468] Processing CreateTablet for tablet ec01f69277c349f491c074a00d015af0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.880467  4101 raft_consensus.cc:383] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:21.880959  4101 raft_consensus.cc:738] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:21.881306  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec01f69277c349f491c074a00d015af0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.881654  4097 tablet_replica.cc:331] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:21.881614  4101 consensus_queue.cc:260] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.882330  4097 raft_consensus.cc:2241] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.882719  4097 raft_consensus.cc:2270] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.886318  4095 tablet_replica.cc:331] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:21.887107  4095 raft_consensus.cc:2241] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.887554  4095 raft_consensus.cc:2270] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.884613  4097 ts_tablet_manager.cc:1905] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.889547  3155 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet eb18d6cfd9834b8db046798e69a544dd (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:21.888525  4101 ts_tablet_manager.cc:1428] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.000s
    @     0x7ff566cc107e  std::__1::__function::__func<>::operator()() at ??:0
    @           0x59bcd5  std::__1::__function::__value_func<>::operator()() at ??:?
I20250628 07:53:21.895876  4095 ts_tablet_manager.cc:1905] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
    @           0x59bb0a  std::__1::function<>::operator()() at ??:?
I20250628 07:53:21.902650  3742 tablet_service.cc:1468] Processing CreateTablet for tablet ec01f69277c349f491c074a00d015af0 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.903723  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec01f69277c349f491c074a00d015af0. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.912564  4097 ts_tablet_manager.cc:1918] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:21.912918  4097 log.cc:1199] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:21.913583  4097 ts_tablet_manager.cc:1939] T c46915dd9de248679ebd1a9ad02db75b P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:21.915933  3155 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet c46915dd9de248679ebd1a9ad02db75b (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
    @     0x7ff55dffc667  kudu::Thread::SuperviseThread() at ??:0
    @           0x465e60  __tsan_thread_start_func at ??:?
I20250628 07:53:21.924062  3163 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 1ed16b61a11544a8968b81868339a936: Sending DeleteTablet for 3 replicas of tablet 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:21.924198  4095 ts_tablet_manager.cc:1918] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:21.924559  4095 log.cc:1199] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:21.925217  4095 ts_tablet_manager.cc:1939] T c46915dd9de248679ebd1a9ad02db75b P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:21.925438  3163 catalog_manager.cc:6652] ReplaceTablet: tablet 5cf5b2d071654ba290785694361f54ea deleted and replaced by tablet 4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:21.925616  3158 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet eb18d6cfd9834b8db046798e69a544dd (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:21.927027  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 5cf5b2d071654ba290785694361f54ea with delete_type TABLET_DATA_DELETED (replaced by tablet 4b07d17dd8a54a7aa2f5723324ba3f65) from {username='slave'} at 127.0.0.1:36942
I20250628 07:53:21.927424  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet 5cf5b2d071654ba290785694361f54ea with delete_type TABLET_DATA_DELETED (replaced by tablet 4b07d17dd8a54a7aa2f5723324ba3f65) from {username='slave'} at 127.0.0.1:44150
I20250628 07:53:21.927743  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet 5cf5b2d071654ba290785694361f54ea with delete_type TABLET_DATA_DELETED (replaced by tablet 4b07d17dd8a54a7aa2f5723324ba3f65) from {username='slave'} at 127.0.0.1:50030
W20250628 07:53:21.930063  3155 catalog_manager.cc:4895] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): delete failed for tablet c46915dd9de248679ebd1a9ad02db75b because the tablet was not found. No further retry: Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:21.930949  4097 tablet_replica.cc:331] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:21.931695  4097 raft_consensus.cc:2241] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.932066  4097 raft_consensus.cc:2270] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.933131  4095 tablet_replica.cc:331] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:21.933815  4095 raft_consensus.cc:2241] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:21.934269  4095 raft_consensus.cc:2270] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:21.936221  4095 ts_tablet_manager.cc:1905] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.946637  3158 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet c46915dd9de248679ebd1a9ad02db75b (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
W20250628 07:53:21.947316  3158 catalog_manager.cc:4908] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): delete failed for tablet 5cf5b2d071654ba290785694361f54ea because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5cf5b2d071654ba290785694361f54ea already in progress: creating tablet
I20250628 07:53:21.948436  4097 ts_tablet_manager.cc:1905] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:21.960553  4101 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:21.967190  4103 tablet_bootstrap.cc:492] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:21.967583  4103 ts_tablet_manager.cc:1397] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.206s	user 0.006s	sys 0.003s
    @     0x7ff55f67b6db  start_thread at ??:0
I20250628 07:53:21.971311  4101 tablet_bootstrap.cc:654] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.974983  4097 ts_tablet_manager.cc:1918] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:21.977792  3449 tablet_service.cc:1468] Processing CreateTablet for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.978710  3449 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4b07d17dd8a54a7aa2f5723324ba3f65. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.975340  4097 log.cc:1199] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/5cf5b2d071654ba290785694361f54ea
I20250628 07:53:21.980427  4097 ts_tablet_manager.cc:1939] T 5cf5b2d071654ba290785694361f54ea P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:21.976866  3592 tablet_service.cc:1468] Processing CreateTablet for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:21.981976  3592 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4b07d17dd8a54a7aa2f5723324ba3f65. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:21.983085  4096 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:21.980063  4103 raft_consensus.cc:357] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.987648  4103 raft_consensus.cc:383] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:21.987931  4103 raft_consensus.cc:738] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:21.988773  4096 tablet_bootstrap.cc:654] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:21.988655  4103 consensus_queue.cc:260] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:21.994854  3162 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:38214
I20250628 07:53:21.995098  3808 heartbeater.cc:499] Master 127.0.107.189:42143 was elected leader, sending a full tablet report...
W20250628 07:53:22.000204  3162 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
I20250628 07:53:22.001312  3155 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 5cf5b2d071654ba290785694361f54ea (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:22.003070  4103 ts_tablet_manager.cc:1428] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.035s	user 0.007s	sys 0.000s
    @     0x7ff55aa8671f  clone at ??:0
I20250628 07:53:22.010812  4096 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:22.011068  4103 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:22.011207  4096 ts_tablet_manager.cc:1397] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.028s	user 0.008s	sys 0.000s
I20250628 07:53:22.013552  4095 ts_tablet_manager.cc:1918] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20250628 07:53:22.013917  4095 log.cc:1199] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/5cf5b2d071654ba290785694361f54ea
I20250628 07:53:22.013675  4096 raft_consensus.cc:357] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.014349  4096 raft_consensus.cc:383] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.014575  4095 ts_tablet_manager.cc:1939] T 5cf5b2d071654ba290785694361f54ea P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:22.014616  4096 raft_consensus.cc:738] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.015398  4096 consensus_queue.cc:260] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.023772  4096 ts_tablet_manager.cc:1428] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.012s	user 0.007s	sys 0.000s
I20250628 07:53:22.028617  4103 tablet_bootstrap.cc:654] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:22.030791  4101 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:22.031329  4101 ts_tablet_manager.cc:1397] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.071s	user 0.011s	sys 0.000s
I20250628 07:53:22.034286  4096 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:22.033965  4101 raft_consensus.cc:357] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.034621  4101 raft_consensus.cc:383] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.034895  4101 raft_consensus.cc:738] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.035684  4101 consensus_queue.cc:260] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.043413  4103 tablet_bootstrap.cc:492] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:22.043825  4103 ts_tablet_manager.cc:1397] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.033s	user 0.005s	sys 0.011s
I20250628 07:53:22.047356  4101 ts_tablet_manager.cc:1428] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.016s	user 0.001s	sys 0.005s
I20250628 07:53:22.047031  4103 raft_consensus.cc:357] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:22.047853  4103 raft_consensus.cc:383] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.048213  4103 raft_consensus.cc:738] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.048942  4103 consensus_queue.cc:260] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:22.052145  4101 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:22.052155  4096 tablet_bootstrap.cc:654] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:22.058130  4101 tablet_bootstrap.cc:654] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:22.059466  4103 ts_tablet_manager.cc:1428] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.002s
I20250628 07:53:22.064091  4101 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:22.064499  4101 ts_tablet_manager.cc:1397] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.013s	user 0.009s	sys 0.000s
I20250628 07:53:22.066962  4101 raft_consensus.cc:357] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.067606  4101 raft_consensus.cc:383] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.067867  4101 raft_consensus.cc:738] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.068717  4103 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:22.068477  4101 consensus_queue.cc:260] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.074855  4101 ts_tablet_manager.cc:1428] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.010s	user 0.004s	sys 0.003s
I20250628 07:53:22.075455  4096 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:22.075881  4096 ts_tablet_manager.cc:1397] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.042s	user 0.003s	sys 0.006s
I20250628 07:53:22.079515  4096 raft_consensus.cc:357] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.080186  4096 raft_consensus.cc:383] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.080569  4096 raft_consensus.cc:738] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.080904  4103 tablet_bootstrap.cc:654] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:22.081326  4096 consensus_queue.cc:260] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.096472  4104 raft_consensus.cc:491] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:22.097040  4104 raft_consensus.cc:513] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.101655  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:22.102411  3613 raft_consensus.cc:2466] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
I20250628 07:53:22.104729  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:22.105157  3396 leader_election.cc:304] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:22.106159  4102 raft_consensus.cc:2802] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:22.106559  4102 raft_consensus.cc:491] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:22.106937  4102 raft_consensus.cc:3058] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:22.109190  4104 leader_election.cc:290] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
W20250628 07:53:22.110824  3402 leader_election.cc:343] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:22.119812  4102 raft_consensus.cc:513] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.126315  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:22.127000  3613 raft_consensus.cc:3058] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:22.142635  3613 raft_consensus.cc:2466] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:22.144172  3396 leader_election.cc:304] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:22.145227  4104 raft_consensus.cc:2802] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:22.123461  4096 ts_tablet_manager.cc:1428] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.047s	user 0.001s	sys 0.006s
I20250628 07:53:22.161664  4102 leader_election.cc:290] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Requested vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:22.161574  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
W20250628 07:53:22.163136  3402 leader_election.cc:343] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:22.169508  4104 raft_consensus.cc:695] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Becoming Leader. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Running, Role: LEADER
I20250628 07:53:22.170749  4104 consensus_queue.cc:237] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.179159  4103 tablet_bootstrap.cc:492] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:22.179600  4103 ts_tablet_manager.cc:1397] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.111s	user 0.034s	sys 0.042s
I20250628 07:53:22.182180  4103 raft_consensus.cc:357] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.182874  4103 raft_consensus.cc:383] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:22.183145  4103 raft_consensus.cc:738] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:22.183773  4103 consensus_queue.cc:260] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.187216  4103 ts_tablet_manager.cc:1428] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.001s
W20250628 07:53:22.356732  3691 connection.cc:537] server connection from 127.0.0.1:50030 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.356756  3547 connection.cc:537] server connection from 127.0.0.1:44150 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.357306  3819 connection.cc:537] client connection to 127.0.107.189:42143 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.357388  3402 connection.cc:537] server connection from 127.0.0.1:36942 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.358800  3819 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.107.189:42143)
W20250628 07:53:22.359860  3819 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.107.189:42143)
W20250628 07:53:22.360271  4109 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:22.358236 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.189:42143 (local address 127.0.107.131:49461)
0628 07:53:22.359101 (+   865us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:22.359312 (+   211us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":81}
W20250628 07:53:22.361282  3819 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.107.189:42143)
W20250628 07:53:22.361490  3548 connection.cc:537] client connection to 127.0.107.189:42143 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.362341  3400 connection.cc:537] client connection to 127.0.107.189:42143 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.363371  3988 connection.cc:537] server connection from 127.0.0.1:39012 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:22.364393  3819 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.107.189:42143)
I20250628 07:53:22.420266  4104 raft_consensus.cc:491] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:22.420753  4104 raft_consensus.cc:513] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.422704  4104 leader_election.cc:290] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:22.423507  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4b07d17dd8a54a7aa2f5723324ba3f65" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:22.424108  3613 raft_consensus.cc:2466] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
W20250628 07:53:22.424422  3402 leader_election.cc:343] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:22.425331  3396 leader_election.cc:304] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:22.425611  3997 raft_consensus.cc:491] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:22.426040  3997 raft_consensus.cc:513] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.426379  4104 raft_consensus.cc:2802] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:22.426811  4104 raft_consensus.cc:491] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:22.427166  4104 raft_consensus.cc:3058] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:22.427893  3997 leader_election.cc:290] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:22.429404  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:22.430300  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "ec01f69277c349f491c074a00d015af0" candidate_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:22.430998  3613 raft_consensus.cc:2391] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a2ddfadc2e7748c0a5505a4c516406b9 in current term 1: Already voted for candidate 01832989f8e6457a8e6508afa155d558 in this term.
I20250628 07:53:22.432042  3688 leader_election.cc:304] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:22.433158  3997 raft_consensus.cc:3058] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:22.437397  4104 raft_consensus.cc:513] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.439638  3997 raft_consensus.cc:2747] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:22.439991  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4b07d17dd8a54a7aa2f5723324ba3f65" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:22.440613  3613 raft_consensus.cc:3058] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:22.441105  4104 leader_election.cc:290] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Requested vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
W20250628 07:53:22.442811  3402 leader_election.cc:343] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:22.447314  3613 raft_consensus.cc:2466] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:22.449414  3396 leader_election.cc:304] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:22.450347  4104 raft_consensus.cc:2802] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:22.450775  4104 raft_consensus.cc:695] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Becoming Leader. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Running, Role: LEADER
I20250628 07:53:22.453299  4104 consensus_queue.cc:237] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:22.488093  4132 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:22.488593  4132 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:22.491398  4132 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:22.494091  3990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:22.499095  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:22.499534  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:22.499827  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:22.500382  4132 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
I20250628 07:53:22.619215  3997 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:22.619624  3997 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:22.621273  3997 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:22.623631  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:22.624100  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:22.624511  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:22.625051  3997 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:22.946982  4104 consensus_queue.cc:1035] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:22.966574  4139 consensus_queue.cc:1035] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:22.968794  3997 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:22.969219  3997 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:22.971081  3997 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:22.975258  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:22.975984  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:22.976397  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:22.977697  3997 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:23.250125  4147 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:23.250535  4147 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:23.252010  4147 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:23.260931  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:23.261410  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:23.261696  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:23.262194  4147 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
I20250628 07:53:23.271173  4139 consensus_queue.cc:1035] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20250628 07:53:23.284538  3402 consensus_peers.cc:489] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 -> Peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Couldn't send request to peer a2ddfadc2e7748c0a5505a4c516406b9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4b07d17dd8a54a7aa2f5723324ba3f65. This is attempt 1: this message will repeat every 5th retry.
I20250628 07:53:23.780295   430 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
/tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:36455
--webserver_interface=127.0.107.188
--webserver_port=42931
--builtin_ntp_servers=127.0.107.148:46421
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--hive_metastore_uris=thrift://127.0.0.1:34709
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20250628 07:53:23.890965  4156 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:23.891548  4156 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:23.893601  4156 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:23.901233  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:23.901773  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:23.902038  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:23.902601  4156 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
W20250628 07:53:24.129238  4153 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20250628 07:53:24.130015  4153 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20250628 07:53:24.130767  4153 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20250628 07:53:24.164775  4153 flags.cc:425] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20250628 07:53:24.165017  4153 flags.cc:425] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20250628 07:53:24.165347  4153 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20250628 07:53:24.165588  4153 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20250628 07:53:24.165818  4153 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20250628 07:53:24.166045  4153 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20250628 07:53:24.214720  4153 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.107.148:46421
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
--hive_metastore_uris=thrift://127.0.0.1:34709
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.107.188:36455
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.107.188
--webserver_port=42931
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true

Master server version:
kudu 1.18.0-SNAPSHOT
revision f0776864a65c19ebdd909053f3e624e8bcc25cee
build type FASTDEBUG
built by None at 28 Jun 2025 07:43:20 UTC on 5fd53c4cbb9d
build id 6809
TSAN enabled
I20250628 07:53:24.216102  4153 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:53:24.217777  4153 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:53:24.233929  4166 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:53:24.237895  4168 ts_tablet_manager.cc:927] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Initiating tablet copy from peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:24.251372  4168 tablet_copy_client.cc:323] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: tablet copy: Beginning tablet copy session from remote peer at address 127.0.107.129:43251
I20250628 07:53:24.254093  3490 tablet_copy_service.cc:140] P 01832989f8e6457a8e6508afa155d558: Received BeginTabletCopySession request for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 from peer a2ddfadc2e7748c0a5505a4c516406b9 ({username='slave'} at 127.0.107.131:44209)
I20250628 07:53:24.254650  3490 tablet_copy_service.cc:161] P 01832989f8e6457a8e6508afa155d558: Beginning new tablet copy session on tablet 4b07d17dd8a54a7aa2f5723324ba3f65 from peer a2ddfadc2e7748c0a5505a4c516406b9 at {username='slave'} at 127.0.107.131:44209: session id = a2ddfadc2e7748c0a5505a4c516406b9-4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:24.262259  3490 tablet_copy_source_session.cc:215] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Tablet Copy: opened 0 blocks and 1 log segments
I20250628 07:53:24.266167  4168 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4b07d17dd8a54a7aa2f5723324ba3f65. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:24.282405  4168 tablet_copy_client.cc:806] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: tablet copy: Starting download of 0 data blocks...
I20250628 07:53:24.283075  4168 tablet_copy_client.cc:670] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: tablet copy: Starting download of 1 WAL segments...
I20250628 07:53:24.287340  4168 tablet_copy_client.cc:538] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20250628 07:53:24.299010  4168 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:24.325500  4168 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap replayed 1/1 log segments. Stats: ops{read=1 overwritten=0 applied=1 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20250628 07:53:24.327914  4168 tablet_bootstrap.cc:492] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap complete.
I20250628 07:53:24.328696  4168 ts_tablet_manager.cc:1397] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.030s	user 0.020s	sys 0.005s
I20250628 07:53:24.333154  4168 raft_consensus.cc:357] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:24.334170  4168 raft_consensus.cc:738] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:24.335000  4168 consensus_queue.cc:260] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:24.341420  4168 ts_tablet_manager.cc:1428] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.012s	user 0.007s	sys 0.003s
I20250628 07:53:24.345898  3490 tablet_copy_service.cc:342] P 01832989f8e6457a8e6508afa155d558: Request end of tablet copy session a2ddfadc2e7748c0a5505a4c516406b9-4b07d17dd8a54a7aa2f5723324ba3f65 received from {username='slave'} at 127.0.107.131:44209
I20250628 07:53:24.346418  3490 tablet_copy_service.cc:434] P 01832989f8e6457a8e6508afa155d558: ending tablet copy session a2ddfadc2e7748c0a5505a4c516406b9-4b07d17dd8a54a7aa2f5723324ba3f65 on tablet 4b07d17dd8a54a7aa2f5723324ba3f65 with peer a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:25.190481  3762 raft_consensus.cc:1215] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 0.0->[1.1-1.1]   Dedup: 1.1->[]
I20250628 07:53:25.229360  4187 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:25.239353  4187 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:25.290555  4187 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
W20250628 07:53:25.311638  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:25.315914  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455): Network error: Client connection negotiation failed: client connection to 127.0.107.188:36455: connect: Connection refused (error 111)
I20250628 07:53:25.316504  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f; no voters: 1ed16b61a11544a8968b81868339a936, fe360a66f1a24378a030a69807008e4d
I20250628 07:53:25.317485  4187 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
I20250628 07:53:25.565944  3863 master_runner.cc:186] Error getting master registration for 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111), 
I20250628 07:53:25.566377  3863 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:25.758453  3995 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:25.759223  3995 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
W20250628 07:53:25.764382  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:25.765645  3995 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:53:25.769544  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:25.770365  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:25.771813  3995 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:53:25.632337  4164 debug-util.cc:398] Leaking SignalData structure 0x7b0800012f00 after lost signal to thread 4153
W20250628 07:53:24.236990  4165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:25.874675  4153 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.639s	user 0.416s	sys 0.881s
W20250628 07:53:25.875358  4153 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.640s	user 0.417s	sys 0.882s
W20250628 07:53:25.877012  4169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:53:25.879879  4167 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1640 milliseconds
I20250628 07:53:25.879897  4153 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:53:25.881235  4153 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20250628 07:53:25.883620  4153 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20250628 07:53:25.884980  4153 hybrid_clock.cc:648] HybridClock initialized: now 1751097205884949 us; error 36 us; skew 500 ppm
I20250628 07:53:25.885818  4153 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:53:25.897390  4153 webserver.cc:469] Webserver started at http://127.0.107.188:42931/ using document root <none> and password file <none>
I20250628 07:53:25.898407  4153 fs_manager.cc:362] Metadata directory not provided
I20250628 07:53:25.898653  4153 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:53:25.907063  4153 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
I20250628 07:53:25.911864  4197 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20250628 07:53:25.912990  4153 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
I20250628 07:53:25.913424  4153 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data,/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
uuid: "fe360a66f1a24378a030a69807008e4d"
format_stamp: "Formatted at 2025-06-28 07:53:07 on dist-test-slave-kfnw"
I20250628 07:53:25.915418  4153 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:53:25.980729  4153 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20250628 07:53:25.982196  4153 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:53:25.982656  4153 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:53:26.057271  4153 rpc_server.cc:307] RPC server started. Bound to: 127.0.107.188:36455
I20250628 07:53:26.057340  4249 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.107.188:36455 every 8 connection(s)
I20250628 07:53:26.060171  4153 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/master-2/data/info.pb
I20250628 07:53:26.063611   430 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskVIvbNg/build/tsan/bin/kudu as pid 4153
I20250628 07:53:26.073745  4250 sys_catalog.cc:263] Verifying existing consensus state
I20250628 07:53:26.080008  4250 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Bootstrap starting.
I20250628 07:53:26.086032  4253 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:26.088330  4253 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:26.093209  4253 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:26.093241  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:26.094244  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:26.094648  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:26.095351  4253 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:26.102162  4254 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:26.102627  4254 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:26.104692  4254 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:26.112201  4250 log.cc:826] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Log is configured to *not* fsync() on all Append() calls
W20250628 07:53:26.126845  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:26.207383  4250 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Bootstrap replayed 1/1 log segments. Stats: ops{read=27 overwritten=0 applied=26 ignored=0} inserts{seen=15 ignored=0} mutations{seen=22 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20250628 07:53:26.208508  4250 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: Bootstrap complete.
I20250628 07:53:26.233404  4250 raft_consensus.cc:357] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:26.242919  4250 raft_consensus.cc:738] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe360a66f1a24378a030a69807008e4d, State: Initialized, Role: FOLLOWER
I20250628 07:53:26.243930  4250 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 26, Last appended: 5.27, Last appended by leader: 27, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:26.247076  4260 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 5 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.248035  4260 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:26.248801  4250 sys_catalog.cc:564] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:53:26.249027  4225 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 7 candidate_status { last_received { term: 6 index: 37 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d" is_pre_election: true
I20250628 07:53:26.249953  4225 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 61070f3974844b798ec0e9a7c3c68f2f in term 5.
I20250628 07:53:26.251149  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936
I20250628 07:53:26.252420  4254 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Leader pre-election won for term 7
I20250628 07:53:26.252789  4254 raft_consensus.cc:491] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Starting leader election (detected failure of leader 1ed16b61a11544a8968b81868339a936)
I20250628 07:53:26.253597  4254 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 6 FOLLOWER]: Advancing to term 7
I20250628 07:53:26.262742  4254 raft_consensus.cc:513] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:26.267762  4225 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "61070f3974844b798ec0e9a7c3c68f2f" candidate_term: 7 candidate_status { last_received { term: 6 index: 37 } } ignore_live_leader: false dest_uuid: "fe360a66f1a24378a030a69807008e4d"
I20250628 07:53:26.268229  4254 leader_election.cc:290] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 election: Requested vote from peers 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143), fe360a66f1a24378a030a69807008e4d (127.0.107.188:36455)
I20250628 07:53:26.268272  4225 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 5 FOLLOWER]: Advancing to term 7
W20250628 07:53:26.268959  3990 leader_election.cc:336] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:26.276355  4225 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 61070f3974844b798ec0e9a7c3c68f2f in term 7.
I20250628 07:53:26.277483  3990 leader_election.cc:304] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 61070f3974844b798ec0e9a7c3c68f2f, fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936
I20250628 07:53:26.278424  4254 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 7 FOLLOWER]: Leader election won for term 7
I20250628 07:53:26.304270  4254 raft_consensus.cc:695] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [term 7 LEADER]: Becoming Leader. State: Replica: 61070f3974844b798ec0e9a7c3c68f2f, State: Running, Role: LEADER
I20250628 07:53:26.305420  4254 consensus_queue.cc:237] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 37, Committed index: 37, Last appended: 6.37, Last appended by leader: 37, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:26.311609  4275 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:26.311854  4275 catalog_manager.cc:1554] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: loading cluster ID for follower catalog manager: success
I20250628 07:53:26.320077  4275 catalog_manager.cc:1576] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: acquiring CA information for follower catalog manager: success
I20250628 07:53:26.326833  3515 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:26.332125  4275 catalog_manager.cc:1604] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20250628 07:53:26.337039  3807 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:26.345302  4214 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" instance_seqno: 1751097196032695) as {username='slave'} at 127.0.107.131:37713; Asking this server to re-register.
I20250628 07:53:26.345978  4277 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 61070f3974844b798ec0e9a7c3c68f2f. Latest consensus state: current_term: 7 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.346853  4277 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: LEADER
I20250628 07:53:26.349104  4283 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:53:26.351383  3807 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:26.343710  4215 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "01832989f8e6457a8e6508afa155d558" instance_seqno: 1751097192145070) as {username='slave'} at 127.0.107.129:56929; Asking this server to re-register.
I20250628 07:53:26.354806  3515 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:26.355734  4214 ts_manager.cc:194] Registered new tserver with Master: a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:26.358392  4214 ts_manager.cc:194] Registered new tserver with Master: 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
I20250628 07:53:26.358901  4283 catalog_manager.cc:671] Loaded metadata for table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]
I20250628 07:53:26.360707  4283 catalog_manager.cc:671] Loaded metadata for table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]
I20250628 07:53:26.362402  4283 catalog_manager.cc:671] Loaded metadata for table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]
I20250628 07:53:26.370349  4283 tablet_loader.cc:96] loaded metadata for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a])
I20250628 07:53:26.371981  4283 tablet_loader.cc:96] loaded metadata for tablet 92b31985c938493cbaca3f1868f92d06 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb])
I20250628 07:53:26.374780  4283 tablet_loader.cc:96] loaded metadata for tablet ec01f69277c349f491c074a00d015af0 (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24])
I20250628 07:53:26.376171  4283 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:53:26.378978  3658 heartbeater.cc:344] Connected to a master server at 127.0.107.188:36455
I20250628 07:53:26.380451  4283 catalog_manager.cc:1261] Loaded cluster ID: eb6a514ccd024ac79e1f7c87e8b97dd6
I20250628 07:53:26.380623  4283 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:53:26.384927  4283 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:53:26.390105  4215 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" instance_seqno: 1751097194108611) as {username='slave'} at 127.0.107.130:40271; Asking this server to re-register.
I20250628 07:53:26.396049  3658 heartbeater.cc:461] Registering TS with master...
I20250628 07:53:26.396489  4283 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Loaded TSK: 0
I20250628 07:53:26.397863  4283 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:53:26.399989  4215 ts_manager.cc:194] Registered new tserver with Master: 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:26.407234  4283 catalog_manager.cc:1529] Loading latest processed Hive Metastore notification log event ID...
I20250628 07:53:26.412676  4283 catalog_manager.cc:5804] Last processed Hive Metastore notification event ID: -1
I20250628 07:53:26.388933  4153 master_runner.cc:186] Error getting master registration for 127.0.107.189:42143: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111), 
I20250628 07:53:26.418346  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:26.432358  4225 raft_consensus.cc:1273] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Refusing update from remote peer 61070f3974844b798ec0e9a7c3c68f2f: Log matching property violated. Preceding OpId in replica: term: 5 index: 27. Preceding OpId from leader: term: 7 index: 39. (index mismatch)
I20250628 07:53:26.435474  4254 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 38, Last known committed idx: 26, Time since last communication: 0.000s
W20250628 07:53:26.443589  3990 consensus_peers.cc:489] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f -> Peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Couldn't send request to peer 1ed16b61a11544a8968b81868339a936. Status: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20250628 07:53:26.446660  4007 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38994:
name: "default.Table_802e2a3d96344b1294c73fbd013d9f26"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:26.463537  4006 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38994:
name: "default.Table_76b9e7d39c4a477ea1ec1a67f687c40a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:26.467401  4004 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:38994:
table { table_name: "default.Table_3e27178fec5347c9a4af581281fd709d" } new_table_name: "default.Table_d9d428eecaeb4f1bb6e7e38a01d3b352" modify_external_catalogs: true
F20250628 07:53:26.470636  4059 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_3e27178fec5347c9a4af581281fd709d, message={"server":"","servicePrincipal":"","db":"default","table":"table_3e27178fec5347c9a4af581281fd709d","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_3e27178fec5347c9a4af581281fd709d\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_3e27178fec5347c9a4af581281fd709d\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_3e27178fec5347c9a4af581281fd709d\",\"kudu.table_id\":\"baf2e85df61f4e19b493ede819300fdb\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1751097196, eventType=CREATE_TABLE, dbName=default, tableName=table_9fe83fcbeebb4e11b040d49634a59618, message={"server":"","servicePrincipal":"","db":"default","table":"table_9fe83fcbeebb4e11b040d49634a59618","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_9fe83fcbeebb4e11b040d49634a59618\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1751097196},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/warehouse/table_9fe83fcbeebb4e11b040d49634a59618\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_9fe83fcbeebb4e11b040d49634a59618\",\"kudu.table_id\":\"8d922997ba434783b26f9d52c4d65f65\",\"transient_lastDdlTime\":\"1751097196\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"eb6a514ccd024ac79e1f7c87e8b97dd6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1751097196,"files":[]}, messageFormat=json-0.2, catName=hive);
*** Check failure stack trace: ***
I20250628 07:53:26.473273  4005 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38994:
name: "default.Table_b76be2ebb8f54a57b15af30be3c83d29"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20250628 07:53:26.475855  4003 catalog_manager.cc:3502] Servicing AlterTable request from {username='slave'} at 127.0.0.1:38994:
table { table_name: "default.Table_104a5469114742b4b7c6e00eac8e93ac" } new_table_name: "default.Table_27ce0e6e2f5240a6b3ce43824e882ef0" modify_external_catalogs: true
I20250628 07:53:26.477156  4002 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:38994
I20250628 07:53:26.477862  4000 master_service.cc:948] ReplaceTablet: received request to replace tablet 4b07d17dd8a54a7aa2f5723324ba3f65 from {username='slave'} at 127.0.0.1:38994
I20250628 07:53:26.479825  4001 catalog_manager.cc:2482] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:38994:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
I20250628 07:53:26.480587  4001 catalog_manager.cc:2730] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:38994:
table { table_name: "default.Table_df57d88234e749dbb8a349ec4eb097f1" } modify_external_catalogs: true
I20250628 07:53:26.487206  4002 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38994:
name: "default.Table_14ae0d9fd13c4744ba376a84483ed2d7"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
    @     0x7f0384611950  google::LogMessage::Flush() at ??:0
I20250628 07:53:26.512965  4260 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 61070f3974844b798ec0e9a7c3c68f2f. Latest consensus state: current_term: 7 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.513758  4260 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:26.519213  4288 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 7 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.520027  4288 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:26.526171  4259 mvcc.cc:204] Tried to move back new op lower bound from 7172494139217203200 to 7172494134050013184. Current Snapshot: MvccSnapshot[applied={T|T < 7172494139217203200 or (T in {7172494139217203200})}]
I20250628 07:53:26.527143  4254 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 7 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.527935  4254 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f [sys.catalog]: This master's current role is: LEADER
    @     0x7f03846116bf  google::LogMessage::~LogMessage() at ??:0
I20250628 07:53:26.535687  4260 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 7 leader_uuid: "61070f3974844b798ec0e9a7c3c68f2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } } }
I20250628 07:53:26.536505  4260 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [sys.catalog]: This master's current role is: FOLLOWER
I20250628 07:53:26.545157  4000 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Sending DeleteTablet for 3 replicas of tablet 4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:26.558848  4000 catalog_manager.cc:6652] ReplaceTablet: tablet 4b07d17dd8a54a7aa2f5723324ba3f65 deleted and replaced by tablet 0426d9be2caf46c9bd5aa93834f38e16
    @     0x7f038f4ee63c  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20250628 07:53:26.606918  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 with delete_type TABLET_DATA_DELETED (replaced by tablet 0426d9be2caf46c9bd5aa93834f38e16) from {username='slave'} at 127.0.0.1:60108
I20250628 07:53:26.607574  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 with delete_type TABLET_DATA_DELETED (replaced by tablet 0426d9be2caf46c9bd5aa93834f38e16) from {username='slave'} at 127.0.0.1:47968
I20250628 07:53:26.608601  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 0426d9be2caf46c9bd5aa93834f38e16 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.609915  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0426d9be2caf46c9bd5aa93834f38e16. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.614094  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 4b07d17dd8a54a7aa2f5723324ba3f65 with delete_type TABLET_DATA_DELETED (replaced by tablet 0426d9be2caf46c9bd5aa93834f38e16) from {username='slave'} at 127.0.0.1:54710
I20250628 07:53:26.615651  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 0426d9be2caf46c9bd5aa93834f38e16 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.617276  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0426d9be2caf46c9bd5aa93834f38e16. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.621620  3450 tablet_service.cc:1468] Processing CreateTablet for tablet 0426d9be2caf46c9bd5aa93834f38e16 (DEFAULT_TABLE table=default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.622738  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0426d9be2caf46c9bd5aa93834f38e16. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.643095  4302 tablet_replica.cc:331] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:26.648180  4302 raft_consensus.cc:2241] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:26.649216  4302 raft_consensus.cc:2270] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.651149  4302 ts_tablet_manager.cc:1905] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.653751  4301 tablet_replica.cc:331] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:26.655084  4301 raft_consensus.cc:2241] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:26.655707  4301 raft_consensus.cc:2270] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.646010  4303 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:26.657712  4300 tablet_replica.cc:331] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:26.658605  4300 raft_consensus.cc:2241] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:26.659519  4300 raft_consensus.cc:2270] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.663636  4300 ts_tablet_manager.cc:1905] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.674315  4301 ts_tablet_manager.cc:1905] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.674539  4305 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:26.683619  4303 tablet_bootstrap.cc:654] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:26.683897  4304 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:26.693559  4000 master_service.cc:948] ReplaceTablet: received request to replace tablet 67e3047db2274d6aa27b7f277f730437 from {username='slave'} at 127.0.0.1:38994
I20250628 07:53:26.703555  4302 ts_tablet_manager.cc:1918] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.704113  4302 log.cc:1199] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:26.706548  4302 ts_tablet_manager.cc:1939] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:26.709116  4305 tablet_bootstrap.cc:654] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:26.710172  3991 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
    @     0x7f038f4eddfc  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
I20250628 07:53:26.723145  4304 tablet_bootstrap.cc:654] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:26.702090  4301 ts_tablet_manager.cc:1918] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.726244  4301 log.cc:1199] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:26.727298  4301 ts_tablet_manager.cc:1939] T 4b07d17dd8a54a7aa2f5723324ba3f65 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:26.728294  4303 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:26.728199  4300 ts_tablet_manager.cc:1918] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.728607  4300 log.cc:1199] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/4b07d17dd8a54a7aa2f5723324ba3f65
I20250628 07:53:26.728766  4303 ts_tablet_manager.cc:1397] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.083s	user 0.014s	sys 0.025s
I20250628 07:53:26.729583  4300 ts_tablet_manager.cc:1939] T 4b07d17dd8a54a7aa2f5723324ba3f65 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:26.731387  4303 raft_consensus.cc:357] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.732019  4303 raft_consensus.cc:383] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:26.732283  4303 raft_consensus.cc:738] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:26.732368  3988 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:26.733335  4305 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:26.732942  4303 consensus_queue.cc:260] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.733704  4305 ts_tablet_manager.cc:1397] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.060s	user 0.008s	sys 0.009s
I20250628 07:53:26.734644  3991 catalog_manager.cc:4928] TS a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): tablet 4b07d17dd8a54a7aa2f5723324ba3f65 (table default.table_df57d88234e749dbb8a349ec4eb097f1 [id=342f6d03d8f840d79fefc55cc112166a]) successfully deleted
I20250628 07:53:26.737846  4305 raft_consensus.cc:357] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.738597  4305 raft_consensus.cc:383] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:26.738919  4305 raft_consensus.cc:738] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:26.739784  4305 consensus_queue.cc:260] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.748142  4304 tablet_bootstrap.cc:492] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:26.748569  4304 ts_tablet_manager.cc:1397] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.072s	user 0.004s	sys 0.010s
I20250628 07:53:26.751130  4304 raft_consensus.cc:357] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.751782  4304 raft_consensus.cc:383] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:26.752023  4304 raft_consensus.cc:738] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:26.756636  4305 ts_tablet_manager.cc:1428] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.023s	user 0.004s	sys 0.002s
I20250628 07:53:26.758373  4304 consensus_queue.cc:260] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.763490  3660 heartbeater.cc:499] Master 127.0.107.190:46841 was elected leader, sending a full tablet report...
W20250628 07:53:26.770408  4000 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
W20250628 07:53:26.770735  4000 catalog_manager.cc:5415] Ignoring report from unknown tablet 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:26.772382  4304 ts_tablet_manager.cc:1428] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.024s	user 0.007s	sys 0.002s
I20250628 07:53:26.773710  4000 catalog_manager.cc:5582] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 01832989f8e6457a8e6508afa155d558 (127.0.107.129). New cstate: current_term: 1 leader_uuid: "01832989f8e6457a8e6508afa155d558" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } }
I20250628 07:53:26.782472  4303 ts_tablet_manager.cc:1428] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.053s	user 0.010s	sys 0.035s
    @     0x7f038f4f1e72  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
I20250628 07:53:26.800138  3809 heartbeater.cc:499] Master 127.0.107.190:46841 was elected leader, sending a full tablet report...
I20250628 07:53:26.803925  3517 heartbeater.cc:499] Master 127.0.107.190:46841 was elected leader, sending a full tablet report...
W20250628 07:53:26.805341  3998 catalog_manager.cc:5415] Ignoring report from unknown tablet c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:26.805681  3998 catalog_manager.cc:5415] Ignoring report from unknown tablet 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:26.806118  3998 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
W20250628 07:53:26.812516  4000 catalog_manager.cc:5415] Ignoring report from unknown tablet 67e3047db2274d6aa27b7f277f730437
I20250628 07:53:26.813934  3998 master_service.cc:948] ReplaceTablet: received request to replace tablet ec01f69277c349f491c074a00d015af0 from {username='slave'} at 127.0.0.1:38994
    @     0x7f038f4f1e2a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
I20250628 07:53:26.835041  3998 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Sending DeleteTablet for 3 replicas of tablet ec01f69277c349f491c074a00d015af0
I20250628 07:53:26.836825  3450 tablet_service.cc:1515] Processing DeleteTablet for tablet ec01f69277c349f491c074a00d015af0 with delete_type TABLET_DATA_DELETED (replaced by tablet 1863ce38a8c64d419ecd69c83ff68827) from {username='slave'} at 127.0.0.1:54710
I20250628 07:53:26.837607  4302 tablet_replica.cc:331] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:26.838337  4302 raft_consensus.cc:2241] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:26.839221  4302 raft_consensus.cc:2270] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.841116  4302 ts_tablet_manager.cc:1905] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
    @     0x7f038f4f1dba  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
I20250628 07:53:26.850622  3593 tablet_service.cc:1515] Processing DeleteTablet for tablet ec01f69277c349f491c074a00d015af0 with delete_type TABLET_DATA_DELETED (replaced by tablet 1863ce38a8c64d419ecd69c83ff68827) from {username='slave'} at 127.0.0.1:60108
I20250628 07:53:26.851413  4300 tablet_replica.cc:331] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:26.851687  3998 catalog_manager.cc:6652] ReplaceTablet: tablet ec01f69277c349f491c074a00d015af0 deleted and replaced by tablet 1863ce38a8c64d419ecd69c83ff68827
I20250628 07:53:26.852144  4300 raft_consensus.cc:2241] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:26.852643  4300 raft_consensus.cc:2270] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.852974  3741 tablet_service.cc:1515] Processing DeleteTablet for tablet ec01f69277c349f491c074a00d015af0 with delete_type TABLET_DATA_DELETED (replaced by tablet 1863ce38a8c64d419ecd69c83ff68827) from {username='slave'} at 127.0.0.1:47968
I20250628 07:53:26.853305  4311 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:26.853802  4301 tablet_replica.cc:331] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:26.853763  4311 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:26.854605  4300 ts_tablet_manager.cc:1905] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.854605  4301 raft_consensus.cc:2241] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:26.855075  4301 raft_consensus.cc:2270] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.855727  4311 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
I20250628 07:53:26.857064  4301 ts_tablet_manager.cc:1905] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
W20250628 07:53:26.859681  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:26.860616  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:26.860970  4302 ts_tablet_manager.cc:1918] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.860981  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:26.861341  4302 log.cc:1199] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/ec01f69277c349f491c074a00d015af0
I20250628 07:53:26.861985  4311 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:26.862277  4302 ts_tablet_manager.cc:1939] T ec01f69277c349f491c074a00d015af0 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:26.868660  4300 ts_tablet_manager.cc:1918] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.869141  4300 log.cc:1199] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/ec01f69277c349f491c074a00d015af0
I20250628 07:53:26.869964  4300 ts_tablet_manager.cc:1939] T ec01f69277c349f491c074a00d015af0 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:26.872583  3988 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet ec01f69277c349f491c074a00d015af0 (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:26.873528  3991 catalog_manager.cc:4928] TS 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): tablet ec01f69277c349f491c074a00d015af0 (table default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]) successfully deleted
I20250628 07:53:26.875214  3450 tablet_service.cc:1468] Processing CreateTablet for tablet 1863ce38a8c64d419ecd69c83ff68827 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.876303  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1863ce38a8c64d419ecd69c83ff68827. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.878640  3593 tablet_service.cc:1468] Processing CreateTablet for tablet 1863ce38a8c64d419ecd69c83ff68827 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.878640  3741 tablet_service.cc:1468] Processing CreateTablet for tablet 1863ce38a8c64d419ecd69c83ff68827 (DEFAULT_TABLE table=default.table_104a5469114742b4b7c6e00eac8e93ac [id=ae56e06624ea49a4a34d86e10d48ca24]), partition=RANGE (key) PARTITION UNBOUNDED
    @     0x7f038f4f1d82  std::__1::__function::__alloc_func<>::operator()() at ??:0
I20250628 07:53:26.879715  3741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1863ce38a8c64d419ecd69c83ff68827. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.879711  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1863ce38a8c64d419ecd69c83ff68827. 1 dirs total, 0 dirs full, 0 dirs failed
    @     0x7f038f4f107e  std::__1::__function::__func<>::operator()() at ??:0
I20250628 07:53:26.901341  4305 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:26.901674  3998 master_service.cc:948] ReplaceTablet: received request to replace tablet 92b31985c938493cbaca3f1868f92d06 from {username='slave'} at 127.0.0.1:38994
    @           0x59bcd5  std::__1::__function::__value_func<>::operator()() at ??:?
    @           0x59bb0a  std::__1::function<>::operator()() at ??:?
I20250628 07:53:26.908943  4305 tablet_bootstrap.cc:654] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:26.922696  3998 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 61070f3974844b798ec0e9a7c3c68f2f: Sending DeleteTablet for 3 replicas of tablet 92b31985c938493cbaca3f1868f92d06
I20250628 07:53:26.924096  3998 catalog_manager.cc:6652] ReplaceTablet: tablet 92b31985c938493cbaca3f1868f92d06 deleted and replaced by tablet dec27c3d369947438522166a0c8ec481
    @     0x7f038682c667  kudu::Thread::SuperviseThread() at ??:0
    @           0x465e60  __tsan_thread_start_func at ??:?
I20250628 07:53:26.927839  3742 tablet_service.cc:1515] Processing DeleteTablet for tablet 92b31985c938493cbaca3f1868f92d06 with delete_type TABLET_DATA_DELETED (replaced by tablet dec27c3d369947438522166a0c8ec481) from {username='slave'} at 127.0.0.1:47968
I20250628 07:53:26.927870  3592 tablet_service.cc:1515] Processing DeleteTablet for tablet 92b31985c938493cbaca3f1868f92d06 with delete_type TABLET_DATA_DELETED (replaced by tablet dec27c3d369947438522166a0c8ec481) from {username='slave'} at 127.0.0.1:60108
I20250628 07:53:26.928555  3449 tablet_service.cc:1515] Processing DeleteTablet for tablet 92b31985c938493cbaca3f1868f92d06 with delete_type TABLET_DATA_DELETED (replaced by tablet dec27c3d369947438522166a0c8ec481) from {username='slave'} at 127.0.0.1:54710
I20250628 07:53:26.928849  4300 tablet_replica.cc:331] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: stopping tablet replica
I20250628 07:53:26.929312  4302 tablet_replica.cc:331] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: stopping tablet replica
I20250628 07:53:26.929829  4300 raft_consensus.cc:2241] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:53:26.930001  4302 raft_consensus.cc:2241] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:26.930524  4302 raft_consensus.cc:2270] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.930935  4300 raft_consensus.cc:2270] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:26.932492  4302 ts_tablet_manager.cc:1905] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.933320  4300 ts_tablet_manager.cc:1905] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:26.935017  4303 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:26.949239  4305 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:26.949707  4305 ts_tablet_manager.cc:1397] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.049s	user 0.012s	sys 0.006s
I20250628 07:53:26.953565  4303 tablet_bootstrap.cc:654] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:26.952685  4305 raft_consensus.cc:357] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.956733  4305 raft_consensus.cc:383] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:26.957021  4305 raft_consensus.cc:738] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:26.960009  4305 consensus_queue.cc:260] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:26.965502  4300 ts_tablet_manager.cc:1918] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.965885  4300 log.cc:1199] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-1/wal/wals/92b31985c938493cbaca3f1868f92d06
I20250628 07:53:26.966609  4300 ts_tablet_manager.cc:1939] T 92b31985c938493cbaca3f1868f92d06 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Deleting consensus metadata
I20250628 07:53:26.968739  4305 ts_tablet_manager.cc:1428] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.019s	user 0.007s	sys 0.000s
I20250628 07:53:26.969626  3988 catalog_manager.cc:4928] TS 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): tablet 92b31985c938493cbaca3f1868f92d06 (table default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]) successfully deleted
I20250628 07:53:26.970815  3593 tablet_service.cc:1468] Processing CreateTablet for tablet dec27c3d369947438522166a0c8ec481 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.970822  3742 tablet_service.cc:1468] Processing CreateTablet for tablet dec27c3d369947438522166a0c8ec481 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.971772  3593 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dec27c3d369947438522166a0c8ec481. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:53:26.971859  3742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dec27c3d369947438522166a0c8ec481. 1 dirs total, 0 dirs full, 0 dirs failed
    @     0x7f0387eab6db  start_thread at ??:0
I20250628 07:53:26.973692  3450 tablet_service.cc:1468] Processing CreateTablet for tablet dec27c3d369947438522166a0c8ec481 (DEFAULT_TABLE table=default.table_3e27178fec5347c9a4af581281fd709d [id=baf2e85df61f4e19b493ede819300fdb]), partition=RANGE (key) PARTITION UNBOUNDED
I20250628 07:53:26.974766  3450 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dec27c3d369947438522166a0c8ec481. 1 dirs total, 0 dirs full, 0 dirs failed
    @     0x7f03832b671f  clone at ??:0
I20250628 07:53:26.995687  4301 ts_tablet_manager.cc:1918] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:26.996050  4301 log.cc:1199] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/ec01f69277c349f491c074a00d015af0
I20250628 07:53:26.996729  4301 ts_tablet_manager.cc:1939] T ec01f69277c349f491c074a00d015af0 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
I20250628 07:53:26.998116  4305 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Bootstrap starting.
I20250628 07:53:27.002256  4304 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:27.004010  4305 tablet_bootstrap.cc:654] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:27.008201  4304 tablet_bootstrap.cc:654] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:27.015417  4304 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:27.015838  4304 ts_tablet_manager.cc:1397] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.014s	user 0.003s	sys 0.008s
I20250628 07:53:27.018265  4304 raft_consensus.cc:357] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.018898  4304 raft_consensus.cc:383] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:27.019181  4304 raft_consensus.cc:738] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:27.019853  4304 consensus_queue.cc:260] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.023417  4304 ts_tablet_manager.cc:1428] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.007s	user 0.001s	sys 0.006s
I20250628 07:53:27.024302  4304 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9: Bootstrap starting.
I20250628 07:53:27.030365  4304 tablet_bootstrap.cc:654] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:27.032532  4310 raft_consensus.cc:491] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:27.032967  4310 raft_consensus.cc:513] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.035055  4310 leader_election.cc:290] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.036963  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0426d9be2caf46c9bd5aa93834f38e16" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:27.037824  3613 raft_consensus.cc:2466] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
I20250628 07:53:27.038976  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0426d9be2caf46c9bd5aa93834f38e16" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:27.039788  4304 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9: No bootstrap required, opened a new log
I20250628 07:53:27.039692  3762 raft_consensus.cc:2466] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
I20250628 07:53:27.040181  4304 ts_tablet_manager.cc:1397] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent bootstrapping tablet: real 0.016s	user 0.011s	sys 0.000s
I20250628 07:53:27.040894  3402 leader_election.cc:304] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
I20250628 07:53:27.041631  4310 raft_consensus.cc:2802] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:27.041962  4310 raft_consensus.cc:491] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:27.042299  4310 raft_consensus.cc:3058] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.042385  4303 tablet_bootstrap.cc:492] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:27.042788  4303 ts_tablet_manager.cc:1397] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.108s	user 0.009s	sys 0.005s
I20250628 07:53:27.045193  4303 raft_consensus.cc:357] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.050398  4305 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: No bootstrap required, opened a new log
I20250628 07:53:27.050217  4310 raft_consensus.cc:513] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.050794  4305 ts_tablet_manager.cc:1397] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent bootstrapping tablet: real 0.053s	user 0.009s	sys 0.012s
I20250628 07:53:27.052703  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0426d9be2caf46c9bd5aa93834f38e16" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:27.052949  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0426d9be2caf46c9bd5aa93834f38e16" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:27.053189  3613 raft_consensus.cc:3058] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.053508  3762 raft_consensus.cc:3058] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.053320  4305 raft_consensus.cc:357] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.054014  4305 raft_consensus.cc:383] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:27.054275  4305 raft_consensus.cc:738] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Initialized, Role: FOLLOWER
I20250628 07:53:27.054983  4305 consensus_queue.cc:260] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.057296  4305 ts_tablet_manager.cc:1428] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.002s
I20250628 07:53:27.059700  3613 raft_consensus.cc:2466] T 0426d9be2caf46c9bd5aa93834f38e16 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:27.060667  3396 leader_election.cc:304] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:27.069149  3762 raft_consensus.cc:2466] T 0426d9be2caf46c9bd5aa93834f38e16 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:27.045809  4303 raft_consensus.cc:383] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:27.077425  4303 raft_consensus.cc:738] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:27.078086  4303 consensus_queue.cc:260] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.078644  4310 leader_election.cc:290] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Requested vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.079152  4310 raft_consensus.cc:2802] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:27.079581  4310 raft_consensus.cc:695] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Becoming Leader. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Running, Role: LEADER
I20250628 07:53:27.080332  4304 raft_consensus.cc:357] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.081059  4304 raft_consensus.cc:383] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:27.081344  4304 raft_consensus.cc:738] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a2ddfadc2e7748c0a5505a4c516406b9, State: Initialized, Role: FOLLOWER
I20250628 07:53:27.081555  4301 tablet_replica.cc:331] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: stopping tablet replica
I20250628 07:53:27.082299  4301 raft_consensus.cc:2241] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20250628 07:53:27.082019  4310 consensus_queue.cc:237] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.082792  4301 raft_consensus.cc:2270] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:53:27.084847  4301 ts_tablet_manager.cc:1905] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:53:27.085927  3995 raft_consensus.cc:491] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:27.086362  3995 raft_consensus.cc:513] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.088129  3995 leader_election.cc:290] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.089690  4302 ts_tablet_manager.cc:1918] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:27.089811  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "dec27c3d369947438522166a0c8ec481" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:27.090047  4302 log.cc:1199] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-0/wal/wals/92b31985c938493cbaca3f1868f92d06
I20250628 07:53:27.090277  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "dec27c3d369947438522166a0c8ec481" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558" is_pre_election: true
I20250628 07:53:27.084048  4304 consensus_queue.cc:260] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.090795  4302 ts_tablet_manager.cc:1939] T 92b31985c938493cbaca3f1868f92d06 P 01832989f8e6457a8e6508afa155d558: Deleting consensus metadata
I20250628 07:53:27.090935  3762 raft_consensus.cc:2466] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 0.
I20250628 07:53:27.092033  3549 leader_election.cc:304] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 
W20250628 07:53:27.092890  3549 leader_election.cc:343] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:27.096329  4304 ts_tablet_manager.cc:1428] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9: Time spent starting tablet: real 0.056s	user 0.000s	sys 0.005s
I20250628 07:53:27.096277  3995 raft_consensus.cc:2802] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:27.096599  3995 raft_consensus.cc:491] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:27.096899  3995 raft_consensus.cc:3058] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.103778  3995 raft_consensus.cc:513] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.110594  4303 ts_tablet_manager.cc:1428] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.068s	user 0.004s	sys 0.002s
I20250628 07:53:27.111867  4303 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558: Bootstrap starting.
I20250628 07:53:27.117893  4303 tablet_bootstrap.cc:654] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558: Neither blocks nor log segments found. Creating new log.
I20250628 07:53:27.133961  3470 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "dec27c3d369947438522166a0c8ec481" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01832989f8e6457a8e6508afa155d558"
W20250628 07:53:27.135162  3549 leader_election.cc:343] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Illegal state: must be running to vote when last-logged opid is not known
I20250628 07:53:27.137295  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "dec27c3d369947438522166a0c8ec481" candidate_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:27.137856  3762 raft_consensus.cc:3058] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.142432  4303 tablet_bootstrap.cc:492] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558: No bootstrap required, opened a new log
I20250628 07:53:27.142853  4303 ts_tablet_manager.cc:1397] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558: Time spent bootstrapping tablet: real 0.031s	user 0.005s	sys 0.005s
I20250628 07:53:27.143910  3995 leader_election.cc:290] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Requested vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.145350  4303 raft_consensus.cc:357] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.146163  4303 raft_consensus.cc:383] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:53:27.146404  4303 raft_consensus.cc:738] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Initialized, Role: FOLLOWER
I20250628 07:53:27.147054  4303 consensus_queue.cc:260] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.149199  4303 ts_tablet_manager.cc:1428] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558: Time spent starting tablet: real 0.006s	user 0.008s	sys 0.000s
I20250628 07:53:27.154800  3762 raft_consensus.cc:2466] T dec27c3d369947438522166a0c8ec481 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3aa2cba1b9aa4da78c59ac8b0877f5aa in term 1.
I20250628 07:53:27.155869  3549 leader_election.cc:304] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa, a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558
I20250628 07:53:27.156910  3995 raft_consensus.cc:2802] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:27.161170  3995 raft_consensus.cc:695] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 LEADER]: Becoming Leader. State: Replica: 3aa2cba1b9aa4da78c59ac8b0877f5aa, State: Running, Role: LEADER
I20250628 07:53:27.161994  3995 consensus_queue.cc:237] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:27.163075  4301 ts_tablet_manager.cc:1918] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:53:27.163439  4301 log.cc:1199] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting WAL directory at /tmp/dist-test-taskVIvbNg/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1751097116344954-430-0/minicluster-data/ts-2/wal/wals/92b31985c938493cbaca3f1868f92d06
I20250628 07:53:27.166210  4301 ts_tablet_manager.cc:1939] T 92b31985c938493cbaca3f1868f92d06 P a2ddfadc2e7748c0a5505a4c516406b9: Deleting consensus metadata
W20250628 07:53:27.286988  3689 connection.cc:537] server connection from 127.0.0.1:47968 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.287917  3546 connection.cc:537] server connection from 127.0.0.1:60108 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.288667  3402 connection.cc:537] server connection from 127.0.0.1:54710 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.289767  3691 connection.cc:537] client connection to 127.0.107.190:46841 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.294615  3402 connection.cc:537] client connection to 127.0.107.190:46841 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.299785  3820 connection.cc:537] client connection to 127.0.107.190:46841 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20250628 07:53:27.302000  3820 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.107.190:46841)
W20250628 07:53:27.302740  3820 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.107.190:46841)
W20250628 07:53:27.303395  3820 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.107.190:46841)
W20250628 07:53:27.304523  3820 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.107.190:46841)
W20250628 07:53:27.316172  4284 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:27.291382 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.190:46841 (local address 127.0.107.130:54333)
0628 07:53:27.292248 (+   866us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:27.292292 (+    44us) client_negotiation.cc:174] Beginning negotiation
0628 07:53:27.292522 (+   230us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:53:27.315292 (+ 22770us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":292}
W20250628 07:53:27.322340  4278 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:27.293135 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.190:46841 (local address 127.0.107.131:40091)
0628 07:53:27.293980 (+   845us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:27.294032 (+    52us) client_negotiation.cc:174] Beginning negotiation
0628 07:53:27.294272 (+   240us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:53:27.321462 (+ 27190us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":284}
W20250628 07:53:27.324975  4181 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:27.297806 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.190:46841 (local address 127.0.107.129:37981)
0628 07:53:27.298677 (+   871us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:27.298725 (+    48us) client_negotiation.cc:174] Beginning negotiation
0628 07:53:27.298942 (+   217us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0628 07:53:27.324088 (+ 25146us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":291}
W20250628 07:53:27.349924  4333 negotiation.cc:337] Failed RPC negotiation. Trace:
0628 07:53:27.311171 (+     0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.107.190:46841 (local address 127.0.0.1:52730)
0628 07:53:27.343459 (+ 32288us) negotiation.cc:107] Waiting for socket to connect
0628 07:53:27.343683 (+   224us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":31722,"threads_started":1}
I20250628 07:53:27.462850  4335 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:27.463214  4335 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:27.465468  4335 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:27.468338  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:27.471866  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:27.472239  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:27.472568  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:27.473160  4335 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:27.739022  4340 raft_consensus.cc:491] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:27.739389  4340 raft_consensus.cc:513] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.740744  4340 leader_election.cc:290] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.741719  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1863ce38a8c64d419ecd69c83ff68827" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" is_pre_election: true
I20250628 07:53:27.741895  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1863ce38a8c64d419ecd69c83ff68827" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" is_pre_election: true
I20250628 07:53:27.742173  3613 raft_consensus.cc:2466] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
I20250628 07:53:27.742477  3762 raft_consensus.cc:2466] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 0.
I20250628 07:53:27.743131  3396 leader_election.cc:304] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:27.743744  4340 raft_consensus.cc:2802] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20250628 07:53:27.744027  4340 raft_consensus.cc:491] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20250628 07:53:27.744254  4340 raft_consensus.cc:3058] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.750591  4340 raft_consensus.cc:513] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:27.751926  4340 leader_election.cc:290] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Requested vote from peers 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723), a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211)
I20250628 07:53:27.752601  3613 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1863ce38a8c64d419ecd69c83ff68827" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa"
I20250628 07:53:27.753054  3613 raft_consensus.cc:3058] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.752909  3762 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1863ce38a8c64d419ecd69c83ff68827" candidate_uuid: "01832989f8e6457a8e6508afa155d558" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a2ddfadc2e7748c0a5505a4c516406b9"
I20250628 07:53:27.753424  3762 raft_consensus.cc:3058] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:27.757352  3613 raft_consensus.cc:2466] T 1863ce38a8c64d419ecd69c83ff68827 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:27.758273  3396 leader_election.cc:304] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 
I20250628 07:53:27.758992  4340 raft_consensus.cc:2802] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:53:27.759783  3762 raft_consensus.cc:2466] T 1863ce38a8c64d419ecd69c83ff68827 P a2ddfadc2e7748c0a5505a4c516406b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01832989f8e6457a8e6508afa155d558 in term 1.
I20250628 07:53:27.760118  4340 raft_consensus.cc:695] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [term 1 LEADER]: Becoming Leader. State: Replica: 01832989f8e6457a8e6508afa155d558, State: Running, Role: LEADER
I20250628 07:53:27.761034  4340 consensus_queue.cc:237] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } }
I20250628 07:53:28.032213  3995 consensus_queue.cc:1035] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.051065  3470 raft_consensus.cc:3058] T dec27c3d369947438522166a0c8ec481 P 01832989f8e6457a8e6508afa155d558 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:53:28.059303  3995 consensus_queue.cc:1035] T dec27c3d369947438522166a0c8ec481 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.094029  4340 consensus_queue.cc:1035] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.115964  4351 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:28.116446  4351 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:28.118080  4340 consensus_queue.cc:1035] T 0426d9be2caf46c9bd5aa93834f38e16 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.120184  4351 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:28.228523  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:28.231329  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:28.231832  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:28.232574  4351 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:28.614746  4341 consensus_queue.cc:1035] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.636734  4340 consensus_queue.cc:1035] T 1863ce38a8c64d419ecd69c83ff68827 P 01832989f8e6457a8e6508afa155d558 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20250628 07:53:28.659545   430 master-stress-test.cc:577] Killing test thread 1/10
I20250628 07:53:28.734417  4364 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:28.734896  4364 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:28.736724  4364 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:28.745437  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:28.746089  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:28.746502  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:28.747833  4364 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:30.193850  4387 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:30.194185  4387 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:30.195645  4387 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:30.213282  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:53:30.214871  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:53:30.215327  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:30.216086  4387 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:31.339591  4397 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:31.340025  4397 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:31.341585  4397 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:53:31.343396  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:53:31.343894  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:31.344177  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:31.344694  4397 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:31.421154  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:31.421514  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:31.590088  4408 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:31.590461  4408 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:31.591780  4408 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:31.593427  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:31.593935  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:31.594362  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:31.594902  4408 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:31.617429  4408 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:31.617825  4408 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:31.619287  4408 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:31.620749  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:31.621183  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:31.621522  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:31.622064  4408 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:32.463197  4421 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:32.463613  4421 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:32.465035  4421 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:32.469741  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:32.470079  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:32.470408  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:32.470908  4421 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:34.254277  4453 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:34.254858  4453 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:34.256603  4453 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:34.259219  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20250628 07:53:34.262610  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:34.263617  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:34.264007  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:34.265529  4453 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:36.424170  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:36.424564  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:36.722589  4482 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:36.722983  4482 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:36.724433  4482 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:36.729195  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:36.729646  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:36.729941  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:36.730465  4482 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:37.708312  4434 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:37.708851  4434 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:37.710911  4434 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:53:37.712420  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:53:37.712945  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:37.713286  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:37.713798  4434 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:39.574074  4510 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:39.574658  4510 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:39.576620  4510 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:39.578418  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:39.578972  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:39.579293  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:39.579849  4510 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:40.458312  4519 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:40.458678  4519 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:40.460114  4519 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:40.461511  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:40.461862  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:40.462169  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:40.462669  4519 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:41.427100  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:41.427453  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:44.445878  4559 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:44.446499  4559 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:44.448155  4559 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:44.450551  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20250628 07:53:44.453747  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:44.454260  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:44.454577  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:44.455121  4559 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:44.612195  4567 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:44.612897  4567 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:44.614789  4567 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:44.616416  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:44.617025  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:44.617401  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:44.617987  4567 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:46.430423  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:46.430799  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:47.597595  4563 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:47.598251  4563 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:47.599740  4563 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:53:47.601272  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:53:47.601753  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:47.602006  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:47.602560  4563 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:51.433336  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:51.433653  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:53.381211  4670 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:53.381577  4670 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:53.382863  4670 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:53.384560  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:53:53.384960  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:53:53.385254  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:53.385735  4670 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:54.938854  4690 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:54.939467  4690 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:53:54.941866  4690 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:53:54.943197  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:53:54.943655  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:53:54.943964  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:53:54.944538  4690 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:53:56.191977  4706 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:53:56.192471  4706 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:53:56.194011  4706 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:53:56.196389  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:53:56.199084  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:53:56.199491  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:53:56.199791  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:53:56.200304  4706 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:53:56.435912  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:53:56.436443  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:53:59.898293  4745 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:53:59.898845  4745 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:53:59.900755  4745 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:53:59.902356  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:53:59.902846  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:53:59.903100  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:53:59.903617  4745 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:01.439378  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:01.439713  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:05.236804  4831 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:05.237370  4831 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:05.238961  4831 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:05.240612  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:54:05.241158  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:54:05.241547  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:05.242206  4831 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:06.443240  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:06.443668  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:09.430249  4894 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:09.430723  4894 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:09.432502  4894 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:09.434619  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:54:09.436946  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:09.437366  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:09.437628  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:09.438112  4894 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:11.446492  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:11.446843  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:11.643144  4917 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:11.643543  4917 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:11.645422  4917 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:11.647028  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:54:11.647349  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:54:11.647722  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:11.648277  4917 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:14.198122  4938 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:14.198740  4938 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:14.200577  4938 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:14.206296  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:54:14.206784  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:54:14.207095  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:14.207660  4938 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
W20250628 07:54:16.382256  3660 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (103 consecutive failures): Network error: Failed to ping master at 127.0.107.190:46841: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:54:16.450074  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:16.450587  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:54:16.780090  3516 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (124 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:54:16.924402  3808 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (121 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:18.001996  4911 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:18.002558  4911 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:54:18.004480  4911 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:54:18.005924  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:54:18.006382  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:54:18.006625  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:54:18.007166  4911 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:18.807917  5003 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:18.808419  5003 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:18.810032  5003 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:18.811411  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:54:18.811914  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:54:18.812186  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:18.812711  5003 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:21.453958  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:21.454334  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:24.895442  5087 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:24.895983  5087 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:24.897564  5087 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:24.899823  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20250628 07:54:24.902889  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:24.903405  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:24.903676  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:24.904147  5087 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:24.956261  5093 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:24.956871  5093 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:24.959385  5093 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:24.960583  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:54:24.961197  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:54:24.961515  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:24.962069  5093 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:26.180190  5110 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:26.180645  5110 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:26.182443  5110 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:26.183862  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:54:26.184242  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:54:26.184598  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:26.185294  5110 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:26.453405  5115 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:26.453786  5115 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:26.455397  5115 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
I20250628 07:54:26.458101  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:26.458714  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:54:26.461357  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:26.461763  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:26.462046  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:26.462461  5115 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:26.776324  5121 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:26.776870  5121 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:54:26.778545  5121 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:54:26.779917  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:54:26.780432  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:54:26.780731  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:54:26.781419  5121 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:27.287565  5126 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:27.287952  5126 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:27.289327  5126 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:27.293884  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:27.294458  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:27.294735  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:27.295274  5126 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:31.461429  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:31.461781  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:31.835397  5180 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:31.835764  5180 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:54:31.837153  5180 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:54:31.838516  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:54:31.838961  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:54:31.839208  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:54:31.839655  5180 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:32.947551  5196 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:32.947904  5196 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:32.949644  5196 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:32.951197  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:54:32.951769  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:54:32.952069  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:32.952607  5196 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:36.464910  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:36.465385  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:36.775909  5250 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:36.776386  5250 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:36.777979  5250 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:36.779464  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 5 similar messages]
W20250628 07:54:36.782548  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:36.783113  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:36.783366  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:36.783867  5250 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:40.826828  5308 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:40.827375  5308 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:40.828953  5308 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:40.830694  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:54:40.831139  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:54:40.831430  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:40.831916  5308 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:41.468261  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:41.468619  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:43.316443  5344 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:43.316839  5344 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:43.318703  5344 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:43.320839  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:54:43.321271  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:54:43.321527  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:43.321990  5344 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:43.618764  5330 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:43.619318  5330 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:54:43.621312  5330 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:54:43.622653  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:54:43.623128  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:54:43.623383  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:54:43.623900  5330 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:43.891959  5351 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:54:43.892350  5351 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:54:43.893776  5351 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:54:43.896476  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:54:43.899045  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:54:43.899529  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:54:43.899785  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:54:43.900252  5351 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:54:46.471750  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:46.472184  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:54:47.903705  4246 debug-util.cc:398] Leaking SignalData structure 0x7b080006f620 after lost signal to thread 4160
W20250628 07:54:47.904255  4246 debug-util.cc:398] Leaking SignalData structure 0x7b08000879e0 after lost signal to thread 4249
I20250628 07:54:48.211951  5416 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:48.212368  5416 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:54:48.213976  5416 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:54:48.217881  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:54:48.218436  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:54:48.218735  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:54:48.219659  5416 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:51.514155  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:51.514532  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:56.517935  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:54:56.518371  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:54:58.318727  5532 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:58.319279  5532 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:58.321346  5532 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:58.322806  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:54:58.323235  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:54:58.323511  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:58.324003  5532 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:54:59.723752  5547 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:54:59.724125  5547 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:54:59.725572  5547 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:54:59.726966  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:54:59.727279  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:54:59.727547  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:54:59.728003  5547 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:01.521073  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:01.521468  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:02.841611  5589 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:02.841971  5589 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:02.843420  5589 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:02.844599  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:55:02.845202  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:55:02.845543  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:02.846028  5589 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:03.042229  5591 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:55:03.042718  5591 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:55:03.044533  5591 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:55:03.046809  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:55:03.050091  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:55:03.050473  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:55:03.050779  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:55:03.051326  5591 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:55:05.134402  5590 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:05.135010  5590 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:55:05.137027  5590 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:55:05.138599  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:55:05.139057  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:55:05.139295  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:55:05.139806  5590 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:55:06.404260  3654 debug-util.cc:398] Leaking SignalData structure 0x7b0800040ea0 after lost signal to thread 3526
W20250628 07:55:06.405128  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000dc980 after lost signal to thread 3657
I20250628 07:55:06.524920  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:06.525386  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:55:07.471825  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000db240 after lost signal to thread 3364
W20250628 07:55:07.472738  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000bff60 after lost signal to thread 3514
I20250628 07:55:09.046670  5654 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:55:09.047151  5654 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:55:09.049122  5654 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:55:09.052739  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:55:09.056334  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:55:09.057001  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:55:09.057444  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:55:09.058243  5654 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
W20250628 07:55:09.609591  3803 debug-util.cc:398] Leaking SignalData structure 0x7b0800041700 after lost signal to thread 3669
W20250628 07:55:09.610536  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000e4100 after lost signal to thread 3806
I20250628 07:55:11.528264  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:11.528626  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:11.851591  5681 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:11.852080  5681 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:11.853734  5681 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:11.855342  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:55:11.855798  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:55:11.856093  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:11.856554  5681 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:16.532148  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:16.532568  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:16.546445  5724 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:16.546911  5724 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:16.548760  5724 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:16.550304  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:55:16.550833  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:55:16.551103  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:16.551625  5724 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:55:16.593576  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000e1280 after lost signal to thread 3669
W20250628 07:55:16.594427  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000db1a0 after lost signal to thread 3806
W20250628 07:55:16.765306  3659 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (241 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:55:17.184043  3516 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (244 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:55:17.352906  3809 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (224 consecutive failures): Network error: Failed to ping master at 127.0.107.190:46841: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:55:17.561905  5733 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:17.562451  5733 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:55:17.564119  5733 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:55:17.565802  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:55:17.566308  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:55:17.566550  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:55:17.567091  5733 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:55:20.520380  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000c88a0 after lost signal to thread 3526
W20250628 07:55:20.521257  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000cd5e0 after lost signal to thread 3657
I20250628 07:55:21.535306  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:21.535714  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:23.363186  5816 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:55:23.363597  5816 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:55:23.365379  5816 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:55:23.367638  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:55:23.370214  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:55:23.370669  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:55:23.370956  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:55:23.371451  5816 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:55:24.443351  5794 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:24.443835  5794 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:55:24.445760  5794 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:55:24.447041  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:55:24.447520  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:55:24.447830  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:55:24.448311  5794 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:26.147141  5855 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:26.147759  5855 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:26.149570  5855 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:26.151170  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:55:26.151670  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:55:26.151937  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:26.152390  5855 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:26.267982  4204 maintenance_manager.cc:419] P fe360a66f1a24378a030a69807008e4d: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033339
I20250628 07:55:26.362658  4203 maintenance_manager.cc:643] P fe360a66f1a24378a030a69807008e4d: FlushMRSOp(00000000000000000000000000000000) complete. Timing: real 0.094s	user 0.036s	sys 0.019s Metrics: {"bytes_written":12591,"cfile_init":1,"drs_written":1,"lbm_read_time_us":204,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2289,"lbm_writes_lt_1ms":30,"peak_mem_usage":0,"rows_written":22,"thread_start_us":357,"threads_started":1,"wal-append.queue_time_us":596}
I20250628 07:55:26.364380  4204 maintenance_manager.cc:419] P fe360a66f1a24378a030a69807008e4d: Scheduling UndoDeltaBlockGCOp(00000000000000000000000000000000): 1388 bytes on disk
I20250628 07:55:26.365626  4203 maintenance_manager.cc:643] P fe360a66f1a24378a030a69807008e4d: UndoDeltaBlockGCOp(00000000000000000000000000000000) complete. Timing: real 0.001s	user 0.000s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":195,"lbm_reads_lt_1ms":4}
I20250628 07:55:26.539047  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:26.539451  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:28.439747  5891 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:28.440093  5891 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:28.441372  5891 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:28.442745  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:55:28.443063  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:55:28.443287  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:28.443675  5891 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:55:30.363325  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000cb560 after lost signal to thread 3526
W20250628 07:55:30.364177  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000dad00 after lost signal to thread 3657
I20250628 07:55:31.569245  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:31.569563  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:55:32.177246  4246 debug-util.cc:398] Leaking SignalData structure 0x7b0800087d80 after lost signal to thread 4160
W20250628 07:55:32.177757  4246 debug-util.cc:398] Leaking SignalData structure 0x7b080006f280 after lost signal to thread 4249
I20250628 07:55:35.773535  5975 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:55:35.774107  5975 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:55:35.775837  5975 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:55:35.778683  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:55:35.781774  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:55:35.782193  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:55:35.782433  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:55:35.782838  5975 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:55:36.572531  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:36.572938  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:36.874409  5965 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:36.874989  5965 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:55:36.876642  5965 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:55:36.878204  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:55:36.878695  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:55:36.878921  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:55:36.879503  5965 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:40.936527  6057 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:40.937009  6057 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:40.939298  6057 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:40.940872  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:55:40.941437  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:55:40.941708  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:40.942157  6057 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:41.575414  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:41.575779  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:44.731946  6122 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:44.732357  6122 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:44.733860  6122 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:44.735096  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:55:44.735424  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:55:44.735664  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:44.736095  6122 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:46.579070  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:46.579504  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:49.506453  6184 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:49.507031  6184 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:55:49.509290  6184 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:55:49.510762  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:55:49.511202  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:55:49.511412  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:55:49.511880  6184 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:55:49.675700  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000e72a0 after lost signal to thread 3669
W20250628 07:55:49.676550  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000ce100 after lost signal to thread 3806
I20250628 07:55:51.581972  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:51.582317  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:52.757743  6263 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:55:52.758205  6263 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:55:52.760202  6263 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:55:52.762326  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:55:52.764839  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:55:52.765234  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:55:52.765511  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:55:52.765975  6263 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:55:53.141026  6276 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:53.141497  6276 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:53.142917  6276 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:53.144510  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:55:53.145047  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:55:53.145414  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:53.145972  6276 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:56.586578  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:55:56.587070  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:55:57.042829  6342 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:57.043278  6342 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:57.045529  6342 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:57.046515  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:55:57.046931  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:55:57.047190  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:57.047711  6342 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:55:57.997447  6354 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:55:57.997833  6354 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:55:57.999191  6354 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:55:58.000530  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:55:58.000959  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:55:58.001227  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:55:58.001662  6354 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:01.589576  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:01.589912  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:02.568452  6428 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:02.569000  6428 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:02.570582  6428 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:02.572232  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:02.572769  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:02.572986  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:02.573601  6428 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:06.593581  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:06.594009  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:07.733368  6523 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:07.733940  6523 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:07.735538  6523 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:07.737052  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:56:07.737646  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:56:07.738039  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:07.738546  6523 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:56:09.131417  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000dede0 after lost signal to thread 3364
W20250628 07:56:09.132330  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000acc60 after lost signal to thread 3514
I20250628 07:56:11.596514  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:11.596850  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:12.516559  6597 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:12.517006  6597 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:12.518905  6597 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:12.520999  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:56:12.523882  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:56:12.524295  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:56:12.524552  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:12.525031  6597 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:56:12.652082  6593 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:12.652820  6593 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:12.655323  6593 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:12.656653  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:12.657150  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:12.657400  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:12.657939  6593 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:16.600031  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:16.600452  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:56:17.125871  3659 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (361 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:56:17.544281  3517 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (345 consecutive failures): Network error: Failed to ping master at 127.0.107.190:46841: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:56:17.705924  3808 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (361 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:56:17.990816  6658 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:17.991267  6658 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:17.993150  6658 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:17.994599  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:56:17.995059  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:56:17.995352  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:17.995849  6658 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:20.567096  6683 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:20.567590  6683 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:20.569195  6683 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:20.570815  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:56:20.574118  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:56:20.574626  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:56:20.574883  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:20.575332  6683 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:56:21.522040  6699 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:21.522428  6699 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:21.523828  6699 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:21.525349  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:56:21.525694  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:56:21.526052  3691 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:21.526669  6699 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:21.603266  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:21.603605  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:24.405014  6742 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:24.405450  6742 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:24.406802  6742 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:24.408284  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:56:24.408689  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:56:24.409055  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:24.409667  6742 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:26.280951  6778 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:26.281476  6778 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:26.283293  6778 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:26.285514  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:56:26.288408  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:56:26.288789  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:56:26.289050  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:26.289568  6778 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:56:26.605928  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:26.606323  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:27.049685  6790 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:27.050304  6790 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:27.051889  6790 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:27.053365  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:27.053860  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:27.054121  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:27.054610  6790 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:30.496935  6804 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:30.497406  6804 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:30.499115  6804 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:30.500556  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:30.501101  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:30.501355  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:30.501849  6804 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:31.608494  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:31.608815  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:36.612044  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:36.612468  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:56:38.145397  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7180 after lost signal to thread 3669
W20250628 07:56:38.146595  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000d2f00 after lost signal to thread 3806
I20250628 07:56:39.249914  6945 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:39.250448  6945 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:39.252540  6945 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:39.254154  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:56:39.254611  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:56:39.254909  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:39.255370  6945 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:40.434486  6957 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:40.435012  6957 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:40.436560  6957 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:40.438469  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:56:40.441020  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:56:40.441452  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:56:40.441725  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:40.442210  6957 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:56:41.615015  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:41.615352  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:41.636860  6956 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:41.637478  6956 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:41.639086  6956 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:41.640544  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:41.641033  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:41.641309  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:41.641803  6956 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:42.551836  6973 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:42.552261  6973 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:42.553781  6973 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:42.555258  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:56:42.555665  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:56:42.555955  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:42.556401  6973 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:44.124011  6983 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:44.124380  6983 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:44.125932  6983 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:44.130318  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:56:44.130698  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:56:44.130987  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:44.131435  6983 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
W20250628 07:56:44.857617  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000409a0 after lost signal to thread 3526
W20250628 07:56:44.858431  3654 debug-util.cc:398] Leaking SignalData structure 0x7b08000cb0e0 after lost signal to thread 3657
I20250628 07:56:46.618645  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:46.619097  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:50.076790  7057 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:56:50.077301  7057 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:56:50.078794  7057 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:56:50.080935  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20250628 07:56:50.083367  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:56:50.083856  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:56:50.084132  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:56:50.084605  7057 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:56:51.621891  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:51.622288  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:56:51.764693  7067 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:51.765280  7067 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:56:51.767266  7067 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:56:51.768772  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:56:51.769304  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:56:51.769529  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:56:51.770011  7067 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:56:51.894915  7082 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:56:51.895754  7082 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:56:51.898351  7082 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:56:51.899322  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:56:51.899804  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:56:51.900096  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:56:51.900604  7082 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:56:56.465734  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000e3600 after lost signal to thread 3364
W20250628 07:56:56.466543  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000ce080 after lost signal to thread 3514
I20250628 07:56:56.625615  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:56:56.626080  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:00.766717  7187 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:00.767325  7187 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:00.769515  7187 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:00.771380  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:57:00.771888  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:57:00.772153  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:00.772655  7187 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:01.137692  7192 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:57:01.138186  7192 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:57:01.139734  7192 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:57:01.141827  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:57:01.144881  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:57:01.145514  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:57:01.145869  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:57:01.146339  7192 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:57:01.628023  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:01.628355  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:03.413661  7225 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:03.414059  7225 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:03.415551  7225 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:03.416991  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:57:03.417440  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:57:03.417754  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:03.418231  7225 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:03.910501  7224 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:03.911105  7224 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:03.912783  7224 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:03.914319  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:03.914789  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:03.915011  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:03.915527  7224 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:06.631907  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:06.632318  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:57:09.280108  4246 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3e00 after lost signal to thread 4160
W20250628 07:57:09.280623  4246 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3f60 after lost signal to thread 4249
I20250628 07:57:10.544747  7312 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:10.545337  7312 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:10.546973  7312 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:10.548626  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:57:10.549065  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:57:10.549346  3691 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:10.549860  7312 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:11.644896  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:11.645282  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:12.380069  7317 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:12.380463  7317 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:12.382349  7317 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:12.383811  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:12.384222  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:12.384430  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:12.384848  7317 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:15.988502  7370 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:15.989049  7370 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:15.991271  7370 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:15.992606  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:57:15.993099  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:57:15.993376  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:15.993840  7370 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:16.133540  7375 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:57:16.134047  7375 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:57:16.135496  7375 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:57:16.137960  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:57:16.140703  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:57:16.141111  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:57:16.141392  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:57:16.141849  7375 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:57:16.647683  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:16.648020  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:17.536749  7399 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:17.537176  7399 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:17.538730  7399 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:17.540198  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:57:17.540542  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:57:17.540952  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:17.541627  7399 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:57:17.560758  3660 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (463 consecutive failures): Network error: Failed to ping master at 127.0.107.190:46841: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:57:17.845614  3517 heartbeater.cc:646] Failed to heartbeat to 127.0.107.190:46841 (465 consecutive failures): Network error: Failed to ping master at 127.0.107.190:46841: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:57:18.058662  3808 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (481 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:57:19.520499  7431 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:19.521134  7431 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:19.522716  7431 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:19.524209  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:19.524686  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:19.524910  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:19.525445  7431 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:21.650893  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:21.651242  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:25.070812  7494 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:25.071367  7494 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:25.073033  7494 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:25.074934  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:25.075487  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:25.075723  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:25.076280  7494 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:26.654529  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:26.654973  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:27.993505  7570 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:27.994019  7570 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:27.995656  7570 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:27.997074  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:57:27.997594  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:57:27.997880  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:27.998371  7570 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:30.335124  7609 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:30.335536  7609 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:30.336942  7609 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:30.338404  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:57:30.338754  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:57:30.339051  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:30.339502  7609 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:31.657620  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:31.657980  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:34.328255  7671 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:57:34.328728  7671 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:57:34.330569  7671 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:57:34.332682  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:57:34.335247  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:57:34.335616  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:57:34.335903  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:57:34.336460  7671 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:57:36.661257  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:36.661785  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:41.664299  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:41.664659  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:44.152491  7790 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:44.153067  7790 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:44.155309  7790 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:44.156736  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:44.157236  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:44.157482  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:44.157999  7790 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:45.374774  7806 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:45.375280  7806 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:45.377457  7806 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:45.379132  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:57:45.379664  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:57:45.379963  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:45.380461  7806 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:46.668028  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:46.668466  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:48.449458  7838 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:48.449857  7838 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:48.451472  7838 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:48.452809  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:57:48.453243  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:57:48.453521  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:48.454105  7838 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:49.985389  7829 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:49.985809  7829 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:57:49.987233  7829 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:57:49.988763  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:57:49.989274  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:57:49.989558  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:57:49.990041  7829 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:51.670830  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:51.671134  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:57:53.005062  7905 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:57:53.005584  7905 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:57:53.007427  7905 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:57:53.008883  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:57:53.009316  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:57:53.009624  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:57:53.010099  7905 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:57:53.854007  7922 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:57:53.854445  7922 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:57:53.856189  7922 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:57:53.858618  4201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:57:53.861295  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:57:53.861810  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:57:53.862164  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:57:53.862753  7922 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:57:56.674567  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:57:56.675196  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:58:00.261196  8025 raft_consensus.cc:491] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:58:00.261730  8025 raft_consensus.cc:513] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:58:00.263343  8025 leader_election.cc:290] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:58:00.264926  3691 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
W20250628 07:58:00.265419  3688 leader_election.cc:343] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: c46915dd9de248679ebd1a9ad02db75b
I20250628 07:58:00.265666  3688 leader_election.cc:304] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:58:00.266144  8025 raft_consensus.cc:2747] T c46915dd9de248679ebd1a9ad02db75b P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:58:01.677794  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:58:01.678128  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:58:05.020080  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000befe0 after lost signal to thread 3364
W20250628 07:58:05.021019  3511 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf420 after lost signal to thread 3514
I20250628 07:58:05.364384  8105 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:58:05.364970  8105 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:58:05.367344  8105 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:58:05.368734  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:58:05.369236  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:58:05.369524  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:58:05.370110  8105 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:58:06.682852  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:58:06.683415  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
W20250628 07:58:08.108634  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000d5020 after lost signal to thread 3669
W20250628 07:58:08.109550  3803 debug-util.cc:398] Leaking SignalData structure 0x7b08000dba40 after lost signal to thread 3806
I20250628 07:58:08.194589  8150 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:58:08.195093  8150 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:58:08.196710  8150 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:58:08.198973  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:58:08.201764  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
W20250628 07:58:08.202201  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
I20250628 07:58:08.202483  4202 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:58:08.202993  8150 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:58:09.862382  8163 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:58:09.862944  8163 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:58:09.864986  8163 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:58:09.866343  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:58:09.866955  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:58:09.867219  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:58:09.867741  8163 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:58:11.686167  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:58:11.686497  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:58:13.719044  8226 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:58:13.719406  8226 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:58:13.720732  8226 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:58:13.722441  4202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20250628 07:58:13.725046  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:58:13.725380  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:58:13.725636  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:58:13.726086  8226 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:58:15.049688  8249 raft_consensus.cc:491] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 61070f3974844b798ec0e9a7c3c68f2f)
I20250628 07:58:15.050395  8249 raft_consensus.cc:513] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "61070f3974844b798ec0e9a7c3c68f2f" member_type: VOTER last_known_addr { host: "127.0.107.190" port: 46841 } } peers { permanent_uuid: "1ed16b61a11544a8968b81868339a936" member_type: VOTER last_known_addr { host: "127.0.107.189" port: 42143 } } peers { permanent_uuid: "fe360a66f1a24378a030a69807008e4d" member_type: VOTER last_known_addr { host: "127.0.107.188" port: 36455 } }
I20250628 07:58:15.052464  8249 leader_election.cc:290] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841), 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143)
W20250628 07:58:15.058056  4202 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 61070f3974844b798ec0e9a7c3c68f2f (127.0.107.190:46841): Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111)
W20250628 07:58:15.058589  4201 leader_election.cc:336] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 1ed16b61a11544a8968b81868339a936 (127.0.107.189:42143): Network error: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)
I20250628 07:58:15.058975  4201 leader_election.cc:304] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fe360a66f1a24378a030a69807008e4d; no voters: 1ed16b61a11544a8968b81868339a936, 61070f3974844b798ec0e9a7c3c68f2f
I20250628 07:58:15.059607  8249 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fe360a66f1a24378a030a69807008e4d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
I20250628 07:58:16.006856  8234 raft_consensus.cc:491] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:58:16.007485  8234 raft_consensus.cc:513] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } }
I20250628 07:58:16.009333  8234 leader_election.cc:290] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211), 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251)
W20250628 07:58:16.010955  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a2ddfadc2e7748c0a5505a4c516406b9 (127.0.107.131:42211): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
W20250628 07:58:16.011453  3549 leader_election.cc:343] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 044e6d34b4584986bdebc4197af39e63
I20250628 07:58:16.011678  3549 leader_election.cc:304] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3aa2cba1b9aa4da78c59ac8b0877f5aa; no voters: 01832989f8e6457a8e6508afa155d558, a2ddfadc2e7748c0a5505a4c516406b9
I20250628 07:58:16.012228  8234 raft_consensus.cc:2747] T 044e6d34b4584986bdebc4197af39e63 P 3aa2cba1b9aa4da78c59ac8b0877f5aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20250628 07:58:16.689169  4153 master_runner.cc:186] Error getting master registration for 127.0.107.190:46841: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection refused (error 111), 
I20250628 07:58:16.689584  4153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
I20250628 07:58:16.956002  8284 raft_consensus.cc:491] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20250628 07:58:16.956516  8284 raft_consensus.cc:513] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01832989f8e6457a8e6508afa155d558" member_type: VOTER last_known_addr { host: "127.0.107.129" port: 43251 } } peers { permanent_uuid: "a2ddfadc2e7748c0a5505a4c516406b9" member_type: VOTER last_known_addr { host: "127.0.107.131" port: 42211 } } peers { permanent_uuid: "3aa2cba1b9aa4da78c59ac8b0877f5aa" member_type: VOTER last_known_addr { host: "127.0.107.130" port: 39723 } }
I20250628 07:58:16.958606  8284 leader_election.cc:290] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251), 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723)
W20250628 07:58:16.959868  3691 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 01832989f8e6457a8e6508afa155d558 (127.0.107.129:43251): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
W20250628 07:58:16.960392  3688 leader_election.cc:343] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3aa2cba1b9aa4da78c59ac8b0877f5aa (127.0.107.130:39723): Not found: Tablet not found: 5cf5b2d071654ba290785694361f54ea
I20250628 07:58:16.960628  3688 leader_election.cc:304] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a2ddfadc2e7748c0a5505a4c516406b9; no voters: 01832989f8e6457a8e6508afa155d558, 3aa2cba1b9aa4da78c59ac8b0877f5aa
I20250628 07:58:16.961109  8284 raft_consensus.cc:2747] T 5cf5b2d071654ba290785694361f54ea P a2ddfadc2e7748c0a5505a4c516406b9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20250628 07:58:17.767326  3819 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
W20250628 07:58:17.767902  3819 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
W20250628 07:58:17.768180  3819 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
W20250628 07:58:17.768450  3819 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
F20250628 07:58:17.779733  3827 master-stress-test.cc:297] Check failed: _s.ok() Bad status: Timed out: Error creating table default.Table_802e2a3d96344b1294c73fbd013d9f26 on the master: CreateTable timed out after deadline expired: CreateTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
*** Check failure stack trace: ***
F20250628 07:58:17.781271  3825 master-stress-test.cc:297] Check failed: _s.ok() Bad status: Timed out: Error creating table default.Table_b76be2ebb8f54a57b15af30be3c83d29 on the master: CreateTable timed out after deadline expired: CreateTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
*** Check failure stack trace: ***
*** Aborted at 1751097497 (unix time) try "date -d @1751097497" if you are using GNU date ***
W20250628 07:58:17.790033  3820 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.107.190:46841)
PC: @                0x0 (unknown)
*** SIGABRT (@0x3e8000001ae) received by PID 430 (TID 0x7f7b195fc700) from PID 430; stack trace: ***
    @           0x2dae00 __tsan::CallUserSignalHandler() at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1915
    @           0x2dd2e3 rtl_sigaction() at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:2005
    @     0x7f7b2be71980 (unknown) at ??:0
    @     0x7f7b270b1fb7 gsignal at ??:0
    @     0x7f7b270b3921 abort at ??:0
    @           0x2e0df6 abort at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:263
    @     0x7f7b2a149da4 google::LogMessage::SendToLog() at ??:0
F20250628 07:58:17.781271  3825 master-stress-test.cc:297] Check failed: _s.ok() Bad status: Timed out: Error creating table default.Table_b76be2ebb8f54a57b15af30be3c83d29 on the master: CreateTable timed out after deadline expired: CreateTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)F20250628 07:58:17.813506  3830 master-stress-test.cc:297] Check failed: _s.ok() Bad status: Timed out: AlterTable timed out after deadline expired: AlterTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.107.190:46841,127.0.107.189:42143,127.0.107.188:36455, num_attempts: 756) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.107.190:46841: connect: Connection reset by peer (error 104)
*** Check failure stack trace: ***
==================
WARNING: ThreadSanitizer: data race (pid=430)
  Atomic write of size 8 at 0x7f7b2a3758b8 by thread T32:
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (master-stress-test+0x334de8)
    #1 unsigned long* google::glog_internal_namespace_::sync_val_compare_and_swap<unsigned long*>(unsigned long**, unsigned long*, unsigned long*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/utilities.h:172:10 (libglog.so.1+0x2b285)
    #2 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/signalhandler.cc:296:7 (libglog.so.1+0x2b285)
    #3 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (master-stress-test+0x2dadff)
    #4 google::LogMessage::Fail() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1946:3 (libglog.so.1+0x18da3)
    #5 google::LogMessage::SendToLog() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1920:5 (libglog.so.1+0x18da3)
    #6 google::LogMessage::Flush() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1777:5 (libglog.so.1+0x1994f)
    #7 google::LogMessageFatal::~LogMessageFatal() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:2557:5 (libglog.so.1+0x1ea8a)
    #8 kudu::MasterStressTest::AlterTableThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:328:7 (master-stress-test+0x35d525)
    #9 decltype(*(std::__1::forward<kudu::MasterStressTest_Test_Test*>(fp0)).*fp()) std::__1::__invoke<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846:1 (master-stress-test+0x38ae0d)
    #10 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (master-stress-test+0x38acc0)
    #11 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*> >(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (master-stress-test+0x38a40a)

  Previous read of size 8 at 0x7f7b2a3758b8 by thread T23:
    #0 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/signalhandler.cc:302:38 (libglog.so.1+0x2ca90)
    #1 __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1911 (master-stress-test+0x2dadff)
    #2 google::LogMessage::Fail() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1946:3 (libglog.so.1+0x18da3)
    #3 google::LogMessage::SendToLog() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1920:5 (libglog.so.1+0x18da3)
    #4 google::LogMessage::Flush() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:1777:5 (libglog.so.1+0x1994f)
    #5 google::LogMessageFatal::~LogMessageFatal() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/logging.cc:2557:5 (libglog.so.1+0x1ea8a)
    #6 kudu::MasterStressTest::CreateTableThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc (master-stress-test+0x35cf33)
    #7 decltype(*(std::__1::forward<kudu::MasterStressTest_Test_Test*>(fp0)).*fp()) std::__1::__invoke<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846:1 (master-stress-test+0x38ae0d)
    #8 void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>&, std::__1::__tuple_indices<2ul>) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:280:5 (master-stress-test+0x38acc0)
    #9 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*> >(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:291:5 (master-stress-test+0x38a40a)
    @     0x7f7b2a14a950 google::LogMessage::Flush() at ??:0

  Location is global 'google::(anonymous namespace)::g_entered_thread_id_pointer' of size 8 at 0x7f7b2a3758b8 (libglog.so.1+0x0000002448b8)

  Thread T32 (tid=3830, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (master-stress-test+0x2de775)
    #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (master-stress-test+0x38a39b)
    #2 std::__1::thread::thread<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (master-stress-test+0x38a0f7)
    #3 void std::__1::allocator<std::__1::thread>::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1809:31 (master-stress-test+0x389ffc)
    #4 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::__construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1687:21 (master-stress-test+0x389f93)
    #5 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1538:14 (master-stress-test+0x389ed3)
    #6 void std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::__construct_one_at_end<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:926:5 (master-stress-test+0x389bde)
    #7 std::__1::thread& std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::emplace_back<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1687:9 (master-stress-test+0x35c98a)
    #8 kudu::MasterStressTest_Test_Test::TestBody() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:560:13 (master-stress-test+0x35b876)
    #9 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
    #10 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
    #11 testing::Test::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674:5 (libgtest.so.1.12.1+0x42a31)
    #12 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43d48)
    #13 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
    #14 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
    #15 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
    #16 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
    #17 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
    #18 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (master-stress-test+0x38d1db)
    #19 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (master-stress-test+0x38c0dc)

  Thread T23 (tid=3825, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (master-stress-test+0x2de775)
    #1 std::__1::__libcpp_thread_create(unsigned long*, void* (*)(void*), void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__threading_support:504:10 (master-stress-test+0x38a39b)
    #2 std::__1::thread::thread<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*, void>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:307:16 (master-stress-test+0x38a0f7)
    #3 void std::__1::allocator<std::__1::thread>::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1809:31 (master-stress-test+0x389ffc)
    #4 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::__construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1687:21 (master-stress-test+0x389f93)
    #5 void std::__1::allocator_traits<std::__1::allocator<std::__1::thread> >::construct<std::__1::thread, void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(std::__1::allocator<std::__1::thread>&, std::__1::thread*, void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1538:14 (master-stress-test+0x389ed3)
    #6 void std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::__emplace_back_slow_path<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1670:5 (master-stress-test+0x389d0e)
    #7 std::__1::thread& std::__1::vector<std::__1::thread, std::__1::allocator<std::__1::thread> >::emplace_back<void (kudu::MasterStressTest::*)(), kudu::MasterStressTest_Test_Test*>(void (kudu::MasterStressTest::*&&)(), kudu::MasterStressTest_Test_Test*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1690:9 (master-stress-test+0x35c99a)
    #8 kudu::MasterStressTest_Test_Test::TestBody() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:557:13 (master-stress-test+0x35b806)
    #9 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
    #10 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
    #11 testing::Test::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674:5 (libgtest.so.1.12.1+0x42a31)
    #12 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43d48)
    #13 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
    #14 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
    #15 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
    #16 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
    #17 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
    #18 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (master-stress-test+0x38d1db)
    #19 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (master-stress-test+0x38c0dc)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/glog-0.6.0/src/utilities.h:172:10 in unsigned long* google::glog_internal_namespace_::sync_val_compare_and_swap<unsigned long*>(unsigned long**, unsigned long*, unsigned long*)
==================
    @     0x7f7b2a14fa8b google::LogMessageFatal::~LogMessageFatal() at ??:0
    @           0x35cf34 kudu::MasterStressTest::CreateTableThread() at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:?
    @           0x38ae0e _ZNSt3__18__invokeIMN4kudu16MasterStressTestEFvvEPNS1_26MasterStressTest_Test_TestEJEvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OS7_DpOS8_ at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846
    @           0x38acc1 _ZNSt3__116__thread_executeINS_10unique_ptrINS_15__thread_structENS_14default_deleteIS2_EEEEMN4kudu16MasterStressTestEFvvEJPNS6_26MasterStressTest_Test_TestEEJLm2EEEEvRNS_5tupleIJT_T0_DpT1_EEENS_15__tuple_indicesIJXspT2_EEEE at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:281
    @           0x38a40b std::__1::__thread_proxy<>() at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:?
    @           0x2dacc0 __tsan_thread_start_func at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:959
    @     0x7f7b2be666db start_thread at ??:0
    @     0x7f7b2719471f clone at ??:0
W20250628 07:58:17.898936  3659 heartbeater.cc:646] Failed to heartbeat to 127.0.107.189:42143 (601 consecutive failures): Network error: Failed to ping master at 127.0.107.189:42143: Client connection negotiation failed: client connection to 127.0.107.189:42143: connect: Connection refused (error 111)